aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/doc/src
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2019-04-16 15:08:54 +0200
committerSiri Hansen <[email protected]>2019-04-16 15:08:54 +0200
commit0c722a427538ba33beb5e92c36f20295b88656a2 (patch)
tree1d1bb164f6bd5d43a120e51843f05ac960789c0c /lib/common_test/doc/src
parent7f4fa8b047f805fa6b989fa493ba80a53d7e8751 (diff)
parent54e599a8be99fe41b45c913b59277dd6e3189e27 (diff)
downloadotp-0c722a427538ba33beb5e92c36f20295b88656a2.tar.gz
otp-0c722a427538ba33beb5e92c36f20295b88656a2.tar.bz2
otp-0c722a427538ba33beb5e92c36f20295b88656a2.zip
Merge branch 'siri/ct/fuzzer_support/ERIERL-143/OTP-14746' into maint
* siri/ct/fuzzer_support/ERIERL-143/OTP-14746: [ct] Fix bug with faulty suite name in end_per_suite config [ct] Add {testcase,TC,RepeatProps} syntax for repeating test cases [ct] Add post_groups/2 and post_all/3 hook functions
Diffstat (limited to 'lib/common_test/doc/src')
-rw-r--r--lib/common_test/doc/src/common_test_app.xml17
-rw-r--r--lib/common_test/doc/src/ct_hooks.xml125
-rw-r--r--lib/common_test/doc/src/write_test_chapter.xml18
3 files changed, 145 insertions, 15 deletions
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml
index 7887a2c3ea..081adeaec7 100644
--- a/lib/common_test/doc/src/common_test_app.xml
+++ b/lib/common_test/doc/src/common_test_app.xml
@@ -72,14 +72,15 @@
<fsummary>Returns the list of all test case groups and test cases
in the module.</fsummary>
<type>
- <v>Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v>
+ <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v>
<v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
<v>GroupName = atom()</v>
- <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | default</v>
+ <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] | default</v>
<v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v>
<v>Shuffle = shuffle | {shuffle,Seed}</v>
<v>Seed = {integer(),integer(),integer()}</v>
- <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
<v>N = integer() | forever</v>
<v>Reason = term()</v>
</type>
@@ -91,7 +92,8 @@
test suite module to be executed. This list also specifies the
order the cases and groups are executed by <c>Common Test</c>.
A test case is represented by an atom,
- the name of the test case function. A test case group is
+ the name of the test case function, or a <c>testcase</c> tuple
+ indicating that the test case shall be repeated. A test case group is
represented by a <c>group</c> tuple, where <c>GroupName</c>,
an atom, is the name of the group (defined in
<seealso marker="#Module:groups-0"><c>groups/0</c></seealso>).
@@ -121,12 +123,13 @@
<v>GroupDefs = [Group]</v>
<v>Group = {GroupName,Properties,GroupsAndTestCases}</v>
<v>GroupName = atom()</v>
- <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}]</v>
- <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v>
+ <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v>
+ <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v>
<v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
<v>Shuffle = shuffle | {shuffle,Seed}</v>
<v>Seed = {integer(),integer(),integer()}</v>
- <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
<v>N = integer() | forever</v>
</type>
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index 048552e4bb..b9bc54ff63 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -109,6 +109,131 @@
</func>
<func>
+ <name since="OTP @OTP-14746@">Module:post_groups(SuiteName, GroupDefs) -&gt; NewGroupDefs</name>
+ <fsummary>Called after groups/0.</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>GroupDefs = NewGroupDefs = [Group]</v>
+ <v>Group = {GroupName,Properties,GroupsAndTestCases}</v>
+ <v>GroupName = atom()</v>
+ <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v>
+ <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v>
+ <v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
+ <v>Shuffle = shuffle | {shuffle,Seed}</v>
+ <v>Seed = {integer(),integer(),integer()}</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>N = integer() | forever</v>
+ </type>
+ <desc>
+ <p>OPTIONAL</p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>.
+ It is used to modify the test group definitions, for
+ instance to add or remove groups or change group properties.</p>
+
+ <p><c>GroupDefs</c> is what
+ <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>
+ returned, that is, a list of group definitions.</p>
+
+ <p><c>NewGroupDefs</c> is the possibly modified version of this list.</p>
+
+ <p>This function is called only if the CTH is added before
+ <c>init_per_suite</c> is run. For details, see section
+ <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso>
+ in the User's Guide.</p>
+
+ <p>Notice that for CTHs that are installed by means of the
+ <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso>
+ function, <c>post_groups/2</c> is called before
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ hook function. However, for CTHs that are installed by means
+ of the CT start flag,
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ function is called first.</p>
+
+ <note>
+ <p>Prior to each test execution, Common Test does a
+ simulated test run in order to count test suites, groups
+ and cases for logging purposes. This causes
+ the <c>post_groups/2</c> hook function to always be called
+ twice. For this reason, side effects are best avoided in
+ this callback.</p>
+ </note>
+ </desc>
+ </func>
+
+ <func>
+ <name since="OTP @OTP-14746@">Module:post_all(SuiteName, Return, GroupDefs) -&gt; NewReturn</name>
+ <fsummary>Called after all/0.</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>Return = NewReturn = Tests | {skip,Reason}</v>
+ <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v>
+ <v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
+ <v>GroupName = atom()</v>
+ <v>Properties = GroupProperties | default</v>
+ <v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v>
+ <v>Shuffle = shuffle | {shuffle,Seed}</v>
+ <v>Seed = {integer(),integer(),integer()}</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>N = integer() | forever</v>
+ <v>GroupDefs = NewGroupDefs = [Group]</v>
+ <v>Group = {GroupName,GroupProperties,GroupsAndTestCases}</v>
+ <v>GroupProperties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v>
+ <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v>
+ <v>Reason = term()</v>
+ </type>
+ <desc>
+ <p>OPTIONAL</p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso>.
+ It is used to modify the set of test cases and test group to
+ be executed, for instance to add or remove test cases and
+ groups, change group properties, or even skip all tests in
+ the suite.</p>
+
+ <p><c>Return</c> is what
+ <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso>
+ returned, that is, a list of test cases and groups to be
+ executed, or a tuple <c>{skip,Reason}</c>.</p>
+
+ <p><c>GroupDefs</c> is what
+ <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>
+ or the <c>post_groups/2</c> hook returned, that is, a list
+ of group definitions.</p>
+
+ <p><c>NewReturn</c> is the possibly modified version of <c>Return</c>.</p>
+
+ <p>This function is called only if the CTH is added before
+ <c>init_per_suite</c> is run. For details, see section
+ <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso>
+ in the User's Guide.</p>
+
+ <p>Notice that for CTHs that are installed by means of the
+ <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso>
+ function, <c>post_all/2</c> is called before
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ hook function. However, for CTHs that are installed by means
+ of the CT start flag,
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ function is called first.</p>
+
+ <note>
+ <p>Prior to each test execution, Common Test does a
+ simulated test run in order to count test suites, groups
+ and cases for logging purposes. This causes
+ the <c>post_all/3</c> hook function to always be called
+ twice. For this reason, side effects are best avoided in
+ this callback.</p>
+ </note>
+ </desc>
+ </func>
+
+ <func>
<name since="OTP R14B02">Module:pre_init_per_suite(SuiteName, InitData, CTHState) -&gt; Result</name>
<fsummary>Called before init_per_suite.</fsummary>
<type>
diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml
index 82dc06834f..5eed748b08 100644
--- a/lib/common_test/doc/src/write_test_chapter.xml
+++ b/lib/common_test/doc/src/write_test_chapter.xml
@@ -455,8 +455,10 @@
GroupDefs = [GroupDef]
GroupDef = {GroupName,Properties,GroupsAndTestCases}
GroupName = atom()
- GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase]
- TestCase = atom()</pre>
+ GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase |
+ {testcase,TestCase,TCRepeatProps}]
+ TestCase = atom()
+ TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</pre>
<p><c>GroupName</c> is the name of the group and must be unique within
the test suite module. Groups can be nested, by including a group definition
@@ -464,11 +466,11 @@
<c>Properties</c> is the list of execution
properties for the group. The possible values are as follows:</p>
<pre>
- Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+ Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]
Shuffle = shuffle | {shuffle,Seed}
Seed = {integer(),integer(),integer()}
- RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
- repeat_until_any_ok | repeat_until_any_fail
+ GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+ repeat_until_any_ok | repeat_until_any_fail
N = integer() | forever</pre>
<p><em>Explanations:</em></p>
@@ -481,8 +483,8 @@
Dependencies Between Test Cases and Suites.</p></item>
<tag><c>shuffle</c></tag>
<item><p>The cases in the group are executed in random order.</p></item>
- <tag><c>repeat</c></tag>
- <item><p>Orders <c>Common Test</c> to repeat execution of the cases in the
+ <tag><c>repeat, repeat_until_*</c></tag>
+ <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the
group a given number of times, or until any, or all, cases fail or succeed.</p></item>
</taglist>
@@ -496,7 +498,7 @@
<c>{group,GroupName}</c> to the <c>all/0</c> list.</p>
<p><em>Example:</em></p>
<pre>
- all() -> [testcase1, {group,group1}, testcase2, {group,group2}].</pre>
+ all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].</pre>
<p>Execution properties with a group tuple in
<c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified.