diff options
Diffstat (limited to 'lib/common_test/doc')
| -rw-r--r-- | lib/common_test/doc/src/common_test_app.xml | 17 | ||||
| -rw-r--r-- | lib/common_test/doc/src/ct_hooks.xml | 125 | ||||
| -rw-r--r-- | lib/common_test/doc/src/notes.xml | 85 | ||||
| -rw-r--r-- | lib/common_test/doc/src/write_test_chapter.xml | 18 |
4 files changed, 230 insertions, 15 deletions
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index a3b3f927eb..5fa87901f6 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -72,14 +72,15 @@ <fsummary>Returns the list of all test case groups and test cases in the module.</fsummary> <type> - <v>Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v> + <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v> <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> <v>GroupName = atom()</v> - <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | default</v> + <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] | default</v> <v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v> <v>Shuffle = shuffle | {shuffle,Seed}</v> <v>Seed = {integer(),integer(),integer()}</v> - <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> <v>N = integer() | forever</v> <v>Reason = term()</v> </type> @@ -91,7 +92,8 @@ test suite module to be executed. This list also specifies the order the cases and groups are executed by <c>Common Test</c>. A test case is represented by an atom, - the name of the test case function. A test case group is + the name of the test case function, or a <c>testcase</c> tuple + indicating that the test case shall be repeated. A test case group is represented by a <c>group</c> tuple, where <c>GroupName</c>, an atom, is the name of the group (defined in <seealso marker="#Module:groups-0"><c>groups/0</c></seealso>). @@ -121,12 +123,13 @@ <v>GroupDefs = [Group]</v> <v>Group = {GroupName,Properties,GroupsAndTestCases}</v> <v>GroupName = atom()</v> - <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}]</v> - <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v> + <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v> + <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v> <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> <v>Shuffle = shuffle | {shuffle,Seed}</v> <v>Seed = {integer(),integer(),integer()}</v> - <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> <v>N = integer() | forever</v> </type> diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index 954be0ffba..03dfceaa1f 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -109,6 +109,131 @@ </func> <func> + <name>Module:post_groups(SuiteName, GroupDefs) -> NewGroupDefs</name> + <fsummary>Called after groups/0.</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>GroupDefs = NewGroupDefs = [Group]</v> + <v>Group = {GroupName,Properties,GroupsAndTestCases}</v> + <v>GroupName = atom()</v> + <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v> + <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v> + <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> + <v>Shuffle = shuffle | {shuffle,Seed}</v> + <v>Seed = {integer(),integer(),integer()}</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>N = integer() | forever</v> + </type> + <desc> + <p>OPTIONAL</p> + + <p>This function is called after + <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>. + It is used to modify the test group definitions, for + instance to add or remove groups or change group properties.</p> + + <p><c>GroupDefs</c> is what + <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso> + returned, that is, a list of group definitions.</p> + + <p><c>NewGroupDefs</c> is the possibly modified version of this list.</p> + + <p>This function is called only if the CTH is added before + <c>init_per_suite</c> is run. For details, see section + <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso> + in the User's Guide.</p> + + <p>Notice that for CTHs that are installed by means of the + <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso> + function, <c>post_groups/2</c> is called before + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + hook function. However, for CTHs that are installed by means + of the CT start flag, + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + function is called first.</p> + + <note> + <p>Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the <c>post_groups/2</c> hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.</p> + </note> + </desc> + </func> + + <func> + <name>Module:post_all(SuiteName, Return, GroupDefs) -> NewReturn</name> + <fsummary>Called after all/0.</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Return = NewReturn = Tests | {skip,Reason}</v> + <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v> + <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> + <v>GroupName = atom()</v> + <v>Properties = GroupProperties | default</v> + <v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v> + <v>Shuffle = shuffle | {shuffle,Seed}</v> + <v>Seed = {integer(),integer(),integer()}</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>N = integer() | forever</v> + <v>GroupDefs = NewGroupDefs = [Group]</v> + <v>Group = {GroupName,GroupProperties,GroupsAndTestCases}</v> + <v>GroupProperties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v> + <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v> + <v>Reason = term()</v> + </type> + <desc> + <p>OPTIONAL</p> + + <p>This function is called after + <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso>. + It is used to modify the set of test cases and test group to + be executed, for instance to add or remove test cases and + groups, change group properties, or even skip all tests in + the suite.</p> + + <p><c>Return</c> is what + <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso> + returned, that is, a list of test cases and groups to be + executed, or a tuple <c>{skip,Reason}</c>.</p> + + <p><c>GroupDefs</c> is what + <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso> + or the <c>post_groups/2</c> hook returned, that is, a list + of group definitions.</p> + + <p><c>NewReturn</c> is the possibly modified version of <c>Return</c>.</p> + + <p>This function is called only if the CTH is added before + <c>init_per_suite</c> is run. For details, see section + <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso> + in the User's Guide.</p> + + <p>Notice that for CTHs that are installed by means of the + <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso> + function, <c>post_all/2</c> is called before + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + hook function. However, for CTHs that are installed by means + of the CT start flag, + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + function is called first.</p> + + <note> + <p>Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the <c>post_all/3</c> hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.</p> + </note> + </desc> + </func> + + <func> <name>Module:pre_init_per_suite(SuiteName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_suite.</fsummary> <type> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 7e909b24cd..d1b9bc320a 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,91 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.15.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The test result when a hook function fails is in general + the same as if the function that the hook is associated + with fails. For example, if <c>post_init_per_testcase</c> + fails the result is that the test case is skipped, as is + the case when <c>init_per_testcase</c> fails.This, + however, was earlier not true for timetrap timeouts or + other error situations where the process running the hook + function was killed. This is now corrected, so the error + handling should be the same no matter how the hook + function fails.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15717 Aux Id: ERIERL-334 </p> + </item> + <item> + <p> + In some rare cases, when two common_test nodes used the + same log directory, a timing problem could occur which + caused common_test to crash because it's log cache file + was unexpectedly empty. This is now corrected.</p> + <p> + Own Id: OTP-15758 Aux Id: ERIERL-342 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Two new common_test hook functions are introduced:</p> + <p> + <c>post_groups/2</c>, which is called after + <c>Suite:groups/0</c><br/> <c>post_all/3</c>, which is + called after <c>Suite:all/0</c></p> + <p> + These functions allow modifying the return values from + the <c>groups/0</c> and <c>all/0</c> functions, + respectively.</p> + <p> + A new term, <c>{testcase,TestCase,RepeatProperties}</c> + is now also allowed in the return from <c>all/0</c>. This + can be used for repeating a single test case a specific + number of times, or until it fails or succeeds once.</p> + <p> + Own Id: OTP-14746 Aux Id: ERIERL-143 </p> + </item> + <item> + <p> + OTP internal test improvements.</p> + <p> + Own Id: OTP-15716</p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.15.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.15.4</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 82dc06834f..5eed748b08 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -455,8 +455,10 @@ GroupDefs = [GroupDef] GroupDef = {GroupName,Properties,GroupsAndTestCases} GroupName = atom() - GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase] - TestCase = atom()</pre> + GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase | + {testcase,TestCase,TCRepeatProps}] + TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</pre> <p><c>GroupName</c> is the name of the group and must be unique within the test suite module. Groups can be nested, by including a group definition @@ -464,11 +466,11 @@ <c>Properties</c> is the list of execution properties for the group. The possible values are as follows:</p> <pre> - Properties = [parallel | sequence | Shuffle | {RepeatType,N}] + Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] Shuffle = shuffle | {shuffle,Seed} Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | - repeat_until_any_ok | repeat_until_any_fail + GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | + repeat_until_any_ok | repeat_until_any_fail N = integer() | forever</pre> <p><em>Explanations:</em></p> @@ -481,8 +483,8 @@ Dependencies Between Test Cases and Suites.</p></item> <tag><c>shuffle</c></tag> <item><p>The cases in the group are executed in random order.</p></item> - <tag><c>repeat</c></tag> - <item><p>Orders <c>Common Test</c> to repeat execution of the cases in the + <tag><c>repeat, repeat_until_*</c></tag> + <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the group a given number of times, or until any, or all, cases fail or succeed.</p></item> </taglist> @@ -496,7 +498,7 @@ <c>{group,GroupName}</c> to the <c>all/0</c> list.</p> <p><em>Example:</em></p> <pre> - all() -> [testcase1, {group,group1}, testcase2, {group,group2}].</pre> + all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].</pre> <p>Execution properties with a group tuple in <c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified. |