aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/common_test/doc/src/run_test.xml13
-rw-r--r--lib/common_test/doc/src/run_test_chapter.xml118
-rw-r--r--lib/common_test/src/ct.erl20
3 files changed, 104 insertions, 47 deletions
diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/run_test.xml
index d609c4287f..2f0a94afba 100644
--- a/lib/common_test/doc/src/run_test.xml
+++ b/lib/common_test/doc/src/run_test.xml
@@ -45,6 +45,11 @@
flags start an Erlang node prepared for running Common Test in a
particular mode.</p>
+ <p>There is an interface function that corresponds to this program,
+ called <c>ct:run_test/1</c>, for starting Common Test from the Erlang
+ shell (or an Erlang program). Please see the <c>ct</c> man page for
+ details.</p>
+
<p><c>run_test</c> also accepts Erlang emulator flags. These are used
when <c>run_test</c> calls <c>erl</c> to start the Erlang node
(making it possible to e.g. add directories to the code server path,
@@ -72,6 +77,8 @@
<p>it prints all valid start flags to stdout.</p>
</description>
+ <marker id="run_test"></marker>
+
<section>
<title>Run tests from command line</title>
<pre>
@@ -83,6 +90,7 @@
[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
ConfigString2 and .. and CallbackModuleN ConfigStringN]
[-decrypt_key Key] | [-decrypt_file KeyFile]
+ [-label Label]
[-logdir LogDir]
[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
[-stylesheet CSSFile]
@@ -107,6 +115,7 @@
[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
ConfigString2 and .. and CallbackModuleN ConfigStringN]
[-decrypt_key Key] | [-decrypt_file KeyFile]
+ [-label Label]
[-logdir LogDir]
[-allow_user_terms]
[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
@@ -128,12 +137,12 @@
<title>Run tests in web based GUI</title>
<pre>
run_test -vts [-browser Browser]
+ [-dir TestDir1 TestDir2 .. TestDirN] |
+ [-suite Suite [[-group Group] [-case Case]]]
[-config ConfigFile1 ConfigFile2 .. ConfigFileN]
[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
ConfigString2 and .. and CallbackModuleN ConfigStringN]
[-decrypt_key Key] | [-decrypt_file KeyFile]
- [-dir TestDir1 TestDir2 .. TestDirN] |
- [-suite Suite [[-group Group] [-case Case]]]
[-include InclDir1 InclDir2 .. InclDirN]
[-no_auto_compile]
[-muliply_timetraps Multiplier]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 207df7f5b5..1efff25f5b 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -21,7 +21,7 @@
</legalnotice>
- <title>Running Test Suites</title>
+ <title>Running Tests</title>
<prepared>Peter Andersson, Kenneth Lundin</prepared>
<docno></docno>
<date></date>
@@ -130,6 +130,8 @@
<p>Other flags that may be used with <c>run_test</c>:</p>
<list>
<item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item>
+ <item><c><![CDATA[-label <name_of_test_run>]]></c>, associates the test run with a name that gets printed
+ in the overview HTML log files.</item>
<item><c>-refresh_logs</c>, refreshes the top level HTML index files.</item>
<item><c>-vts</c>, start web based GUI (see below).</item>
<item><c>-shell</c>, start interactive shell mode (see below).</item>
@@ -335,43 +337,72 @@
<marker id="test_specifications"></marker>
<title>Using test specifications</title>
- <p>The most expressive way to specify what to test is to use a so
- called test specification. A test specification is a sequence of
- Erlang terms. The terms may be declared in a text file or passed
- to the test server at runtime as a list (see <c>run_testspec/1</c>
- in the manual page for <c>ct</c>). There are two general types
- of terms: configuration terms and test specification terms.</p>
- <p>With configuration terms it is possible to import configuration
- data (similar to <c>run_test -config</c>), specify HTML log
- directories (similar to <c>run_test -logdir</c>), give aliases
- to test nodes and test directories (to make a specification
- easier to read and maintain), enable code coverage analysis
- (see the <seealso marker="cover_chapter#cover">Code Coverage
- Analysis</seealso> chapter) and specify event_handler plugins
+ <p>The most flexible way to specify what to test, is to use a so
+ called test specification. A test specification is a sequence of
+ Erlang terms. The terms may be declared in a text file or passed
+ to the test server at runtime as a list
+ (see <c>run_testspec/1</c> in the manual page
+ for <c>ct</c>). There are two general types of terms:
+ configuration terms and test specification terms.</p>
+ <p>With configuration terms it is possible to e.g. label the test
+ run (similar to <c>run_test -label</c>), evaluate arbitrary expressions
+ before starting a test, import configuration
+ data (similar to
+ <c>run_test -config/-userconfig</c>), specify HTML log directories (similar
+ to
+ <c>run_test -logdir</c>), give aliases to test nodes and test
+ directories (to make a specification easier to read and
+ maintain), enable code coverage analysis (see
+ the <seealso marker="cover_chapter#cover">Code Coverage
+ Analysis</seealso> chapter) and specify event_handler plugins
(see the <seealso marker="event_handler_chapter#event_handling">
- Event Handling</seealso> chapter). There is also a term
- for specifying include directories that should be passed on
- to the compiler when automatic compilation is performed
- (similar to <c>run_test -include</c>, see above).</p>
- <p>With test specification terms it is possible to state exactly which
- tests should run and in which order. A test term specifies either
- one or more suites or one or more test cases. An arbitrary number of test
- terms may be declared in sequence. A test term can also specify one or
- more test suites or test cases to be skipped. Skipped suites and cases
- are not executed and show up in the HTML test log as SKIPPED.</p>
-
- <note><p>It is not yet possible to specify test case groups in
- test specifications. This will be supported in a soon upcoming
- release.</p></note>
+ Event Handling</seealso> chapter). There is also a term for
+ specifying include directories that should be passed on to the
+ compiler when automatic compilation is performed (similar
+ to <c>run_test -include</c>, see above).</p>
+ <p>With test specification terms it is possible to state exactly
+ which tests should run and in which order. A test term specifies
+ either one or more suites, one or more test case groups, or one
+ or more test cases in a group or suite.</p>
+ <p>An arbitrary number of test terms may be declared in sequence.
+ Common Test will compile the terms into one or more tests to be
+ performed in one resulting test run. Note that a term that
+ specifies a set of test cases will "swallow" one that only
+ specifies a subset of these cases. E.g. the result of merging
+ one term that specifies that all cases in suite S should be
+ executed, with another term specifying only test case X and Y in
+ S, is a test of all cases in S. However, if a term specifying
+ test case X and Y in S is merged with a term specifying case Z
+ in S, the result is a test of X, Y and Z in S.</p>
+ <p>A test term can also specify one or more test suites, groups,
+ or test cases to be skipped. Skipped suites, groups and cases
+ are not executed and show up in the HTML test log files as
+ SKIPPED.</p>
+ <p>When a test case group is specified, the resulting test
+ executes the
+ <c>init_per_group</c> function, followed by all test cases and
+ sub groups (including their configuration functions), and
+ finally the <c>end_per_group</c> function. Also if particular
+ test cases in a group are specified, <c>init_per_group</c>
+ and <c>end_per_group</c> for the group in question are
+ called. If a group which is defined (in <c>Suite:group/0</c>) to
+ be a sub group of another group, is specified (or particular test
+ cases of a sub group are), Common Test will call the configuration
+ functions for the top level groups as well as for the sub group
+ in question (making it possible to pass configuration data all
+ the way from <c>init_per_suite</c> down to the test cases in the
+ sub group).</p>
<p>Below is the test specification syntax. Test specifications can
- be used to run tests both in a single test host environment and in
- a distributed Common Test environment (Large Scale Testing). The init term,
- as well as node parameters, are only relevant in the latter (see the
- <seealso marker="ct_master_chapter#test_specifications">Large Scale Testing</seealso>
- chapter for information). For details on the event_handler term, see the
- <seealso marker="event_handler_chapter#event_handling">Event Handling</seealso>
- chapter.</p>
+ be used to run tests both in a single test host environment and
+ in a distributed Common Test environment (Large Scale
+ Testing). The node parameters in the init term are only
+ relevant in the latter (see the
+ <seealso marker="ct_master_chapter#test_specifications">Large
+ Scale Testing</seealso> chapter for information). For details on
+ the event_handler term, see the
+ <seealso marker="event_handler_chapter#event_handling">Event
+ Handling</seealso> chapter.</p>
<p>Config terms:</p>
<pre>
{node, NodeAlias, Node}.
@@ -379,11 +410,17 @@
{init, InitOptions}.
{init, [NodeAlias], InitOptions}.
+ {label, Label}.
+ {label, NodeRefs, Label}.
+
{multiply_timetraps, N}.
+ {multiply_timetraps, NodeRefs, N}.
+
{scale_timetraps, Bool}.
+ {scale_timetraps, NodeRefs, Bool}.
{cover, CoverSpecFile}.
- {cover, NodeRef, CoverSpecFile}.
+ {cover, NodeRefs, CoverSpecFile}.
{include, IncludeDirs}.
{include, NodeRefs, IncludeDirs}.
@@ -409,6 +446,12 @@
{suites, DirRef, Suites}.
{suites, NodeRefs, DirRef, Suites}.
+ {groups, DirRef, Suite, Groups}.
+ {groups, NodeRefsDirRef, Suite, Groups}.
+
+ {groups, DirRef, Suite, Group, {cases,Cases}}.
+ {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}.
+
{cases, DirRef, Suite, Cases}.
{cases, NodeRefs, DirRef, Suite, Cases}.
@@ -437,6 +480,9 @@
InitArgs = [term()]
DirRef = DirAlias | Dir
Suites = atom() | [atom()] | all
+ Suite = atom()
+ Groups = atom() | [atom()] | all
+ Group = atom()
Cases = atom() | [atom()] | all
Comment = string() | ""
</pre>
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 0d82a86e7d..8ae175f10d 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -138,10 +138,10 @@ run(TestDirs) ->
%%%-----------------------------------------------------------------
%%% @spec run_test(Opts) -> Result
%%% Opts = [OptTuples]
-%%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
-%%% {userconfig, UserConfig} |
-%%% {testcase,Cases} | {group,Groups} | {spec,TestSpecs} |
-%%% {allow_user_terms,Bool} | {logdir,LogDir} |
+%%% OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} |
+%%% {testcase,Cases} | {spec,TestSpecs} | {label,Label} |
+%%% {config,CfgFiles} | {userconfig, UserConfig} |
+%%% {allow_user_terms,Bool} | {logdir,LogDir} |
%%% {silent_connections,Conns} | {stylesheet,CSSFile} |
%%% {cover,CoverSpecFile} | {step,StepOpts} |
%%% {event_handler,EventHandlers} | {include,InclDirs} |
@@ -149,15 +149,16 @@ run(TestDirs) ->
%%% {repeat,N} | {duration,DurTime} | {until,StopTime} |
%%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
%%% {refresh_logs,LogDir} | {basic_html,Bool}
-%%% CfgFiles = [string()] | string()
-%%% UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
-%%% CallbackMod = atom()
-%%% CfgStrings = [string()] | string()
%%% TestDirs = [string()] | string()
%%% Suites = [string()] | string()
%%% Cases = [atom()] | atom()
%%% Groups = [atom()] | atom()
%%% TestSpecs = [string()] | string()
+%%% Label = string() | atom()
+%%% CfgFiles = [string()] | string()
+%%% UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
+%%% CallbackMod = atom()
+%%% CfgStrings = [string()] | string()
%%% LogDir = string()
%%% Conns = all | [atom()]
%%% CSSFile = string()
@@ -177,7 +178,8 @@ run(TestDirs) ->
%%% DecryptFile = string()
%%% Result = [TestResult] | {error,Reason}
%%% @doc Run tests as specified by the combination of options in <code>Opts</code>.
-%%% The options are the same as those used with the <code>run_test</code> program.
+%%% The options are the same as those used with the
+%%% <seealso marker="run_test#run_test"><code>run_test</code></seealso> program.
%%% Note that here a <code>TestDir</code> can be used to point out the path to
%%% a <code>Suite</code>. Note also that the option <code>testcase</code>
%%% corresponds to the <code>-case</code> option in the <code>run_test</code>