From acd87425acf7705328b01aa8f0a804207ffe4790 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Wed, 10 Jan 2018 16:13:52 +0100 Subject: [ct] Add post_groups/2 and post_all/3 hook functions --- lib/common_test/doc/src/ct_hooks.xml | 123 +++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) (limited to 'lib/common_test/doc/src') diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index 954be0ffba..613b694796 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -108,6 +108,129 @@ + + Module:post_groups(SuiteName, GroupDefs) -> NewGroupDefs + Called after groups/0. + + SuiteName = atom() + GroupDefs = NewGroupDefs = [Group] + Group = {GroupName,Properties,GroupsAndTestCases} + GroupName = atom() + Properties = [parallel | sequence | Shuffle | {RepeatType,N}] + GroupsAndTestCases = [Group | {group,GroupName} | TestCase] + TestCase = atom() + Shuffle = shuffle | {shuffle,Seed} + Seed = {integer(),integer(),integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + + +

OPTIONAL

+ +

This function is called after + groups/0. + It is used to modify the test group definitions, for + instance to add or remove groups or change group properties.

+ +

GroupDefs is what + groups/0 + returned, that is, a list of group definitions.

+ +

NewGroupDefs is the possibly modified version of this list.

+ +

This function is called only if the CTH is added before + init_per_suite is run. For details, see section + CTH Scope + in the User's Guide.

+ +

Notice that for CTHs that are installed by means of the + suite/0 + function, post_groups/2 is called before + the init/2 + hook function. However, for CTHs that are installed by means + of the CT start flag, + the init/2 + function is called first.

+ + +

Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the post_groups/2 hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.

+ + + + + + Module:post_all(SuiteName, Return, GroupDefs) -> NewReturn + Called after all/0. + + SuiteName = atom() + Return = NewReturn = Tests | {skip,Reason} + Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] + TestCase = atom() + GroupName = atom() + Properties = GroupProperties | default + SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}] + Shuffle = shuffle | {shuffle,Seed} + Seed = {integer(),integer(),integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + GroupDefs = NewGroupDefs = [Group] + Group = {GroupName,GroupProperties,GroupsAndTestCases} + GroupProperties = [parallel | sequence | Shuffle | {RepeatType,N}] + GroupsAndTestCases = [Group | {group,GroupName} | TestCase] + Reason = term() + + +

OPTIONAL

+ +

This function is called after + all/0. + It is used to modify the set of test cases and test group to + be executed, for instance to add or remove test cases and + groups, change group properties, or even skip all tests in + the suite.

+ +

Return is what + all/0 + returned, that is, a list of test cases and groups to be + executed, or a tuple {skip,Reason}.

+ +

GroupDefs is what + groups/0 + or the post_groups/2 hook returned, that is, a list + of group definitions.

+ +

NewReturn is the possibly modified version of Return.

+ +

This function is called only if the CTH is added before + init_per_suite is run. For details, see section + CTH Scope + in the User's Guide.

+ +

Notice that for CTHs that are installed by means of the + suite/0 + function, post_all/2 is called before + the init/2 + hook function. However, for CTHs that are installed by means + of the CT start flag, + the init/2 + function is called first.

+ + +

Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the post_all/3 hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.

+ + + + Module:pre_init_per_suite(SuiteName, InitData, CTHState) -> Result Called before init_per_suite. -- cgit v1.2.3 From 3d087911ef3ecd769333b822c337bbbb76877871 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Wed, 20 Mar 2019 16:48:41 +0100 Subject: [ct] Add {testcase,TC,RepeatProps} syntax for repeating test cases --- lib/common_test/doc/src/common_test_app.xml | 17 ++++++++++------- lib/common_test/doc/src/ct_hooks.xml | 14 ++++++++------ lib/common_test/doc/src/write_test_chapter.xml | 18 ++++++++++-------- 3 files changed, 28 insertions(+), 21 deletions(-) (limited to 'lib/common_test/doc/src') diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index a3b3f927eb..5fa87901f6 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -72,14 +72,15 @@ Returns the list of all test case groups and test cases in the module. - Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] + Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | default + Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] | default SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}] Shuffle = shuffle | {shuffle,Seed} Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail N = integer() | forever Reason = term() @@ -91,7 +92,8 @@ test suite module to be executed. This list also specifies the order the cases and groups are executed by Common Test. A test case is represented by an atom, - the name of the test case function. A test case group is + the name of the test case function, or a testcase tuple + indicating that the test case shall be repeated. A test case group is represented by a group tuple, where GroupName, an atom, is the name of the group (defined in groups/0). @@ -121,12 +123,13 @@ GroupDefs = [Group] Group = {GroupName,Properties,GroupsAndTestCases} GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType,N}] - GroupsAndTestCases = [Group | {group,GroupName} | TestCase] + Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] + GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}] TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] Shuffle = shuffle | {shuffle,Seed} Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail N = integer() | forever diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index 613b694796..03dfceaa1f 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -116,12 +116,13 @@ GroupDefs = NewGroupDefs = [Group] Group = {GroupName,Properties,GroupsAndTestCases} GroupName = atom() - Properties = [parallel | sequence | Shuffle | {RepeatType,N}] - GroupsAndTestCases = [Group | {group,GroupName} | TestCase] + Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] + GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}] TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] Shuffle = shuffle | {shuffle,Seed} Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail N = integer() | forever @@ -169,18 +170,19 @@ SuiteName = atom() Return = NewReturn = Tests | {skip,Reason} - Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] + Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}] TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}] GroupName = atom() Properties = GroupProperties | default SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}] Shuffle = shuffle | {shuffle,Seed} Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail + GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail N = integer() | forever GroupDefs = NewGroupDefs = [Group] Group = {GroupName,GroupProperties,GroupsAndTestCases} - GroupProperties = [parallel | sequence | Shuffle | {RepeatType,N}] + GroupProperties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] GroupsAndTestCases = [Group | {group,GroupName} | TestCase] Reason = term() diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 82dc06834f..5eed748b08 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -455,8 +455,10 @@ GroupDefs = [GroupDef] GroupDef = {GroupName,Properties,GroupsAndTestCases} GroupName = atom() - GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase] - TestCase = atom() + GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase | + {testcase,TestCase,TCRepeatProps}] + TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]

GroupName is the name of the group and must be unique within the test suite module. Groups can be nested, by including a group definition @@ -464,11 +466,11 @@ Properties is the list of execution properties for the group. The possible values are as follows:

- Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+ Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]
  Shuffle = shuffle | {shuffle,Seed}
  Seed = {integer(),integer(),integer()}
- RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
-              repeat_until_any_ok | repeat_until_any_fail
+ GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+                   repeat_until_any_ok | repeat_until_any_fail
  N = integer() | forever

Explanations:

@@ -481,8 +483,8 @@ Dependencies Between Test Cases and Suites.

shuffle

The cases in the group are executed in random order.

 - repeat -

Orders Common Test to repeat execution of the cases in the + repeat, repeat_until_* +

Orders Common Test to repeat execution of all the cases in the group a given number of times, or until any, or all, cases fail or succeed.

 @@ -496,7 +498,7 @@ {group,GroupName} to the all/0 list.

Example:

- all() -> [testcase1, {group,group1}, testcase2, {group,group2}].
+ all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].

Execution properties with a group tuple in all/0: {group,GroupName,Properties} can also be specified. -- cgit v1.2.3