From 2c3e05dbf52c1e682bf23bfb06664420433e1774 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 2 Jul 2010 11:46:53 +0200 Subject: Add documentation for group specifications and for the label option --- lib/common_test/doc/src/run_test.xml | 13 ++- lib/common_test/doc/src/run_test_chapter.xml | 118 +++++++++++++++++++-------- lib/common_test/src/ct.erl | 20 +++-- 3 files changed, 104 insertions(+), 47 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/run_test.xml index d609c4287f..2f0a94afba 100644 --- a/lib/common_test/doc/src/run_test.xml +++ b/lib/common_test/doc/src/run_test.xml @@ -45,6 +45,11 @@ flags start an Erlang node prepared for running Common Test in a particular mode.

+

There is an interface function that corresponds to this program, + called ct:run_test/1, for starting Common Test from the Erlang + shell (or an Erlang program). Please see the ct man page for + details.

+

run_test also accepts Erlang emulator flags. These are used when run_test calls erl to start the Erlang node (making it possible to e.g. add directories to the code server path, @@ -72,6 +77,8 @@

it prints all valid start flags to stdout.

+ +
Run tests from command line
@@ -83,6 +90,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
 	 ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
 	[-stylesheet CSSFile]
@@ -107,6 +115,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-allow_user_terms]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
@@ -128,12 +137,12 @@
     Run tests in web based GUI
     
         run_test -vts [-browser Browser]
+        [-dir TestDir1 TestDir2 .. TestDirN] |
+        [-suite Suite [[-group Group] [-case Case]]]
 	[-config ConfigFile1 ConfigFile2 .. ConfigFileN]
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
-        [-dir TestDir1 TestDir2 .. TestDirN] |
-        [-suite Suite [[-group Group] [-case Case]]]
 	[-include InclDir1 InclDir2 .. InclDirN]
 	[-no_auto_compile]
 	[-muliply_timetraps Multiplier]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 207df7f5b5..1efff25f5b 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -21,7 +21,7 @@
 
     
 
-    Running Test Suites
+    Running Tests
     Peter Andersson, Kenneth Lundin
     
     
@@ -130,6 +130,8 @@
     

Other flags that may be used with run_test:

]]>, specifies where the HTML log files are to be written. + ]]>, associates the test run with a name that gets printed + in the overview HTML log files. -refresh_logs, refreshes the top level HTML index files. -vts, start web based GUI (see below). -shell, start interactive shell mode (see below). @@ -335,43 +337,72 @@ Using test specifications -

The most expressive way to specify what to test is to use a so - called test specification. A test specification is a sequence of - Erlang terms. The terms may be declared in a text file or passed - to the test server at runtime as a list (see run_testspec/1 - in the manual page for ct). There are two general types - of terms: configuration terms and test specification terms.

-

With configuration terms it is possible to import configuration - data (similar to run_test -config), specify HTML log - directories (similar to run_test -logdir), give aliases - to test nodes and test directories (to make a specification - easier to read and maintain), enable code coverage analysis - (see the Code Coverage - Analysis chapter) and specify event_handler plugins +

The most flexible way to specify what to test, is to use a so + called test specification. A test specification is a sequence of + Erlang terms. The terms may be declared in a text file or passed + to the test server at runtime as a list + (see run_testspec/1 in the manual page + for ct). There are two general types of terms: + configuration terms and test specification terms.

+

With configuration terms it is possible to e.g. label the test + run (similar to run_test -label), evaluate arbitrary expressions + before starting a test, import configuration + data (similar to + run_test -config/-userconfig), specify HTML log directories (similar + to + run_test -logdir), give aliases to test nodes and test + directories (to make a specification easier to read and + maintain), enable code coverage analysis (see + the Code Coverage + Analysis chapter) and specify event_handler plugins (see the - Event Handling chapter). There is also a term - for specifying include directories that should be passed on - to the compiler when automatic compilation is performed - (similar to run_test -include, see above).

-

With test specification terms it is possible to state exactly which - tests should run and in which order. A test term specifies either - one or more suites or one or more test cases. An arbitrary number of test - terms may be declared in sequence. A test term can also specify one or - more test suites or test cases to be skipped. Skipped suites and cases - are not executed and show up in the HTML test log as SKIPPED.

- -

It is not yet possible to specify test case groups in - test specifications. This will be supported in a soon upcoming - release.

+ Event Handling chapter). There is also a term for + specifying include directories that should be passed on to the + compiler when automatic compilation is performed (similar + to run_test -include, see above).

+

With test specification terms it is possible to state exactly + which tests should run and in which order. A test term specifies + either one or more suites, one or more test case groups, or one + or more test cases in a group or suite.

+

An arbitrary number of test terms may be declared in sequence. + Common Test will compile the terms into one or more tests to be + performed in one resulting test run. Note that a term that + specifies a set of test cases will "swallow" one that only + specifies a subset of these cases. E.g. the result of merging + one term that specifies that all cases in suite S should be + executed, with another term specifying only test case X and Y in + S, is a test of all cases in S. However, if a term specifying + test case X and Y in S is merged with a term specifying case Z + in S, the result is a test of X, Y and Z in S.

+

A test term can also specify one or more test suites, groups, + or test cases to be skipped. Skipped suites, groups and cases + are not executed and show up in the HTML test log files as + SKIPPED.

+

When a test case group is specified, the resulting test + executes the + init_per_group function, followed by all test cases and + sub groups (including their configuration functions), and + finally the end_per_group function. Also if particular + test cases in a group are specified, init_per_group + and end_per_group for the group in question are + called. If a group which is defined (in Suite:group/0) to + be a sub group of another group, is specified (or particular test + cases of a sub group are), Common Test will call the configuration + functions for the top level groups as well as for the sub group + in question (making it possible to pass configuration data all + the way from init_per_suite down to the test cases in the + sub group).

Below is the test specification syntax. Test specifications can - be used to run tests both in a single test host environment and in - a distributed Common Test environment (Large Scale Testing). The init term, - as well as node parameters, are only relevant in the latter (see the - Large Scale Testing - chapter for information). For details on the event_handler term, see the - Event Handling - chapter.

+ be used to run tests both in a single test host environment and + in a distributed Common Test environment (Large Scale + Testing). The node parameters in the init term are only + relevant in the latter (see the + Large + Scale Testing chapter for information). For details on + the event_handler term, see the + Event + Handling chapter.

Config terms:

       {node, NodeAlias, Node}.
@@ -379,11 +410,17 @@
       {init, InitOptions}.
       {init, [NodeAlias], InitOptions}.
 
+      {label, Label}.
+      {label, NodeRefs, Label}.
+
       {multiply_timetraps, N}.
+      {multiply_timetraps, NodeRefs, N}.
+
       {scale_timetraps, Bool}.
+      {scale_timetraps, NodeRefs, Bool}.
  
       {cover, CoverSpecFile}.
-      {cover, NodeRef, CoverSpecFile}.
+      {cover, NodeRefs, CoverSpecFile}.
       
       {include, IncludeDirs}.
       {include, NodeRefs, IncludeDirs}.
@@ -409,6 +446,12 @@
       {suites, DirRef, Suites}.                                
       {suites, NodeRefs, DirRef, Suites}.
       
+      {groups, DirRef, Suite, Groups}.
+      {groups, NodeRefsDirRef, Suite, Groups}.
+
+      {groups, DirRef, Suite, Group, {cases,Cases}}.
+      {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}.
+
       {cases, DirRef, Suite, Cases}.                           
       {cases, NodeRefs, DirRef, Suite, Cases}.
 
@@ -437,6 +480,9 @@
       InitArgs      = [term()]
       DirRef        = DirAlias | Dir
       Suites        = atom() | [atom()] | all
+      Suite         = atom()
+      Groups        = atom() | [atom()] | all
+      Group         = atom()
       Cases         = atom() | [atom()] | all
       Comment       = string() | ""
     
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 0d82a86e7d..8ae175f10d 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -138,10 +138,10 @@ run(TestDirs) -> %%%----------------------------------------------------------------- %%% @spec run_test(Opts) -> Result %%% Opts = [OptTuples] -%%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} | -%%% {userconfig, UserConfig} | -%%% {testcase,Cases} | {group,Groups} | {spec,TestSpecs} | -%%% {allow_user_terms,Bool} | {logdir,LogDir} | +%%% OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} | +%%% {testcase,Cases} | {spec,TestSpecs} | {label,Label} | +%%% {config,CfgFiles} | {userconfig, UserConfig} | +%%% {allow_user_terms,Bool} | {logdir,LogDir} | %%% {silent_connections,Conns} | {stylesheet,CSSFile} | %%% {cover,CoverSpecFile} | {step,StepOpts} | %%% {event_handler,EventHandlers} | {include,InclDirs} | @@ -149,15 +149,16 @@ run(TestDirs) -> %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | %%% {refresh_logs,LogDir} | {basic_html,Bool} -%%% CfgFiles = [string()] | string() -%%% UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings} -%%% CallbackMod = atom() -%%% CfgStrings = [string()] | string() %%% TestDirs = [string()] | string() %%% Suites = [string()] | string() %%% Cases = [atom()] | atom() %%% Groups = [atom()] | atom() %%% TestSpecs = [string()] | string() +%%% Label = string() | atom() +%%% CfgFiles = [string()] | string() +%%% UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings} +%%% CallbackMod = atom() +%%% CfgStrings = [string()] | string() %%% LogDir = string() %%% Conns = all | [atom()] %%% CSSFile = string() @@ -177,7 +178,8 @@ run(TestDirs) -> %%% DecryptFile = string() %%% Result = [TestResult] | {error,Reason} %%% @doc Run tests as specified by the combination of options in Opts. -%%% The options are the same as those used with the run_test program. +%%% The options are the same as those used with the +%%% run_test program. %%% Note that here a TestDir can be used to point out the path to %%% a Suite. Note also that the option testcase %%% corresponds to the -case option in the run_test -- cgit v1.2.3