From 7af62a670c8b29050c3c1dcf0f4175ea18377de6 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sat, 10 Dec 2011 22:35:52 +0100 Subject: Document group specifications in all/0 and the group info function --- lib/common_test/doc/src/common_test_app.xml | 222 ++++++++++++++++++------- lib/common_test/doc/src/write_test_chapter.xml | 66 +++++++- 2 files changed, 222 insertions(+), 66 deletions(-) (limited to 'lib/common_test/doc') diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index f58b2ab0a9..c7f6c7ce5c 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -69,12 +69,25 @@ - Module:all() -> TestCases | {skip,Reason} - Returns the list of all test cases in the module. + Module:all() -> Tests | {skip,Reason} + Returns the list of all test case groups and test cases + in the module. - TestCases = [atom() | {group,GroupName}] - Reason = term() + Tests = [TestCase | {group,GroupName} | + {group,GroupName,Properties} | + {group,GroupName,Properties,SubGroups}] + TestCase = atom() GroupName = atom() + Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | + default + SubGroups = [{GroupName,Properties} | + {GroupName,Properties,SubGroups}] + Shuffle = shuffle | {shuffle,Seed} + Seed = {integer(),integer(),integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | + repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + Reason = term() @@ -85,9 +98,14 @@ This list also specifies the order the cases and groups will be executed by Common Test. A test case is represented by an atom, the name of the test case function. A test case group is - represented by a {group,GroupName} tuple, where - GroupName, an atom, is the name of the group (defined - with groups/0).

+ represented by a group tuple, where GroupName, + an atom, is the name of the group (defined in groups/0). + Execution properties for groups may also be specified, both + for a top level group and for any of its sub-groups. + Group execution properties specified here, will override + properties in the group definition (see groups/0). + (With value default, the group definition properties + will be used).

If {skip,Reason} is returned, all test cases in the module will be skipped, and the Reason will @@ -120,14 +138,16 @@

OPTIONAL

-

See Test case +

Function for defining test case groups. Please see + Test case groups in the User's Guide for details.

 Module:suite() -> [Info] - Test suite info function (providing default data for the suite). + Test suite info function (providing default data + for the suite). Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | @@ -160,11 +180,11 @@

This is the test suite info function. It is supposed to return a list of tagged tuples that specify various properties - regarding the execution of this test suite (common for all + related to the execution of this test suite (common for all test cases in the suite).

The timetrap tag sets the maximum time each - test case is allowed to take (including init_per_testcase/2 + test case is allowed to execute (including init_per_testcase/2 and end_per_testcase/2). If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. If a TimeFunc function is specified, @@ -172,9 +192,9 @@ TimeVal format.

The require tag specifies configuration variables - that are required by test cases in the suite. If the required - configuration variables are not found in any of the - configuration files, all test cases are skipped. For more + that are required by test cases (and/or configuration functions) + in the suite. If the required configuration variables are not found + in any of the configuration files, all test cases are skipped. For more information about the 'require' functionality, see the reference manual for the function ct:require/[1,2].

@@ -196,8 +216,9 @@ - Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | {skip_and_save,Reason,SaveConfig} - Test suite initialization. + Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | + {skip_and_save,Reason,SaveConfig} + Test suite initializations. Config = NewConfig = SaveConfig = [{Key,Value}] Key = atom() @@ -208,15 +229,15 @@

OPTIONAL

-

This function is called as the first function in the - suite. It typically contains initialization which is common for +

This configuration function is called as the first function in the + suite. It typically contains initializations which are common for all test cases in the suite, and which shall only be done - once. The Config parameter is the configuration + once. The Config parameter is the configuration data which can be modified here. Whatever is returned from this function is given as Config to all configuration functions and test cases in the suite. If {skip,Reason} - is returned, all test cases in the suite will be skipped and Reason - printed in the overview log for the suite.

+ is returned, all test cases in the suite will be skipped + and Reason printed in the overview log for the suite.

For information on save_config and skip_and_save, please see Dependencies @@ -224,29 +245,106 @@ - - Module:end_per_suite(Config) -> void() | {save_config,SaveConfig} - Test suite finalization. - - Config = SaveConfig = [{Key,Value}] - Key = atom() - Value = term() - + + Module:end_per_suite(Config) -> void() | + {save_config,SaveConfig} + Test suite finalization. + + Config = SaveConfig = [{Key,Value}] + Key = atom() + Value = term() + + + +

OPTIONAL

+

This function is called as the last test case in the + suite. It is meant to be used for cleaning up after + init_per_suite/1. + For information on save_config, please see + Dependencies + between Test Cases and Suites in the User's Guide.

+ + + + + Module:group(GroupName) -> [Info] + Test case group info function (providing default data + for a test case group, i.e. its test cases and sub-groups). + + Info = {timetrap,Time} | {require,Required} | + {require,Name,Required} | {userdata,UserData} | + {silent_connections,Conns} | {stylesheet,CSSFile} | + {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | + {hours,integer()} + TimeFunc = {Mod,Func,Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key,SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | {CTHModule, CTHInitArgs} | + {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + +

OPTIONAL

-

This function is called as the last test case in the - suite. It is meant to be used for cleaning up after init_per_suite/1. - For information on save_config, please see - Dependencies between - Test Cases and Suites in the User's Guide.

- +

This is the test case group info function. It is supposed to + return a list of tagged tuples that specify various properties + related to the execution of a test case group (i.e. its test cases + and sub-groups). Properties set by groups/1 override + properties with the same key that have been previously set by + suite/0.

+ +

The timetrap tag sets the maximum time each + test case is allowed to execute (including init_per_testcase/2 + and end_per_testcase/2). If the timetrap time is + exceeded, the test case fails with reason + timetrap_timeout. If a TimeFunc function is specified, + it will be called initially and must return a value on + TimeVal format.

+ +

The require tag specifies configuration variables + that are required by test cases (and/or configuration functions) + in the suite. If the required configuration variables are not found + in any of the configuration files, all test cases in this group are skipped. + For more information about the 'require' functionality, see the + reference manual for the function + ct:require/[1,2].

+ +

With userdata, it is possible for the user to + specify arbitrary test case group related information which can be + read by calling ct:userdata/2.

+ +

The ct_hooks tag specifies which + Common Test Hooks + are to be run together with this suite.

+ +

Other tuples than the ones defined will simply be ignored.

+ +

For more information about the test case group info function, + see Test + case group info function in the User's Guide.

+ - Module:init_per_group(GroupName, Config) -> NewConfig | {skip,Reason} - Test case group initialization. + Module:init_per_group(GroupName, Config) -> NewConfig | + {skip,Reason} + Test case group initializations. GroupName = atom() Config = NewConfig = [{Key,Value}] @@ -258,16 +356,16 @@

OPTIONAL

-

This function is called before execution of a test case group. - It typically contains initialization which is common for - all test cases in the group, and which shall only be performed - once. GroupName is the name of the group, as specified in - the group definition (see groups/0). The Config - parameter is the configuration which can be modified here. - Whatever is returned from this function is given as Config - to all test cases in the group. If {skip,Reason} is returned, - all test cases in the group will be skipped and Reason printed - in the overview log for the group.

+

This configuration function is called before execution of a + test case group. It typically contains initializations which are + common for all test cases and sub-groups in the group, and which + shall only be performed once. GroupName is the name of the + group, as specified in the group definition (see groups/0). The + Config parameter is the configuration data which can be modified + here. The return value of this function is given as Config + to all test cases and sub-groups in the group. If {skip,Reason} + is returned, all test cases in the group will be skipped and + Reason printed in the overview log for the group.

For information about test case groups, please see Test case @@ -276,7 +374,8 @@ - Module:end_per_group(GroupName, Config) -> void() | {return_group_result,Status} + Module:end_per_group(GroupName, Config) -> void() | + {return_group_result,Status} Test case group finalization. GroupName = atom() @@ -305,7 +404,7 @@ Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail,Reason} | {skip,Reason} - Test case initialization. + Test case initializations. TestCase = atom() Config = NewConfig = [{Key,Value}] @@ -385,10 +484,13 @@

This is the test case info function. It is supposed to return a list of tagged tuples that specify various properties - regarding the execution of this particular test case.

+ related to the execution of this particular test case. + Properties set by Testcase/0 override + properties that have been previously set for the test case + by group/1 or suite/0.

The timetrap tag sets the maximum time the - test case is allowed to take. If the timetrap time is + test case is allowed to execute. If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. init_per_testcase/2 and end_per_testcase/2 are included in the @@ -397,16 +499,16 @@ and must return a value on TimeVal format.

The require tag specifies configuration variables - that are required by the test case. If the required - configuration variables are not found in any of the + that are required by the test case (and/or init/end_per_testcase/2). + If the required configuration variables are not found in any of the configuration files, the test case is skipped. For more information about the 'require' functionality, see the reference manual for the function ct:require/[1,2].

If timetrap and/or require is not set, the - default values specified in the suite/0 return list - will be used.

+ default values specified by suite/0 (or + group/1) will be used.

With userdata, it is possible for the user to specify arbitrary test case related information which can be @@ -438,12 +540,12 @@

This is the implementation of a test case. Here you must call the functions you want to test, and do whatever you need to check the result. If something fails, make sure the - function causes a runtime error, or call ct:fail/[0,1] - (which also causes the test case process to crash).

+ function causes a runtime error, or call ct:fail/1/2 + (which also causes the test case process to terminate).

-

Elements from the Config parameter can be read - with the ?config macro. The config - macro is defined in ct.hrl

+

Elements from the Config list can e.g. be read + with proplists:get_value/2 (or the macro ?config + defined in ct.hrl).

You can return {skip,Reason} if you decide not to run the test case after all. Reason will then be diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index e35888e68f..b8487ea2d4 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -68,7 +68,7 @@

Each test suite module must export the function all/0 which returns the list of all test case groups and test cases - in that module. + to be executed in that module.

@@ -450,11 +450,65 @@ 
       all() -> [testcase1, {group,group1}, testcase2, {group,group2}].
-

Properties may be combined so that e.g. if shuffle, - repeat_until_any_fail and sequence are all specified, the test - cases in the group will be executed repeatedly and in random order until - a test case fails, when execution is immediately stopped and the rest of - the cases skipped.

+

It is also possible to specify execution properties with a group + tuple in all/0: {group,GroupName,Properties}. These + properties will override those specified in the group definition (see + groups/0 above). This way, it's possible to run the same set of tests, + but with different properties, without having to make copies of the group + definition in question.

+ +

If a group contains sub-groups, the execution properties for these may + also be specified in the group tuple: + {group,GroupName,Properties,SubGroups}, where SubGroups + is a list of tuples, {GroupName,Properties}, or + {GroupName,Properties,SubGroups}, representing the sub-groups. + Any sub-groups defined in group/0 for a group, that are not specified + in the SubGroups list, will simply execute with their pre-defined + properties.

+ +

Example:

+ 
+      groups() -> {tests1, [], [{tests2, [], [t2a,t2b]},
+                                {tests3, [], [t31,t3b]}]}.
+

To execute group 'tests1' twice with different properties for 'tests2' + each time:

+ 
+      all() ->
+         [{group, tests1, default, [{tests2, [parallel]}]},
+          {group, tests1, default, [{tests2, [shuffle,{repeat,10}]}]}].
+

Note that this is equivalent to this specification:

+ 
+      all() ->
+         [{group, tests1, default, [{tests2, [parallel]},
+                                    {tests3, default}]},
+          {group, tests1, default, [{tests2, [shuffle,{repeat,10}]},
+                                    {tests3, default}]}].
+

The value default states that the pre-defined properties + should be used.

+

Here's an example of how to override properties in a scenario + with deeply nested groups:

+ 
+      groups() ->
+         [{tests1, [], [{group, tests2}]},
+          {tests2, [], [{group, tests3}]},
+          {tests3, [{repeat,2}], [t3a,t3b,t3c]}].
+
+      all() ->
+         [{group, tests1, default, 
+           [{tests2, default,
+             [{tests3, [parallel,{repeat,100}]}]}]}].
+ +

The syntax described above may also be used in Test Specifications + in order to change properties of groups at the time of execution, + without even having to edit the test suite (please see the + Test + Specifications chapter for more info).

+ +

As illustrated above, properties may be combined. If e.g. + shuffle, repeat_until_any_fail and sequence + are all specified, the test cases in the group will be executed + repeatedly, and in random order, until a test case fails. Then + execution is immediately stopped and the rest of the cases skipped.

Before execution of a group begins, the configuration function init_per_group(GroupName, Config) is called (the function is -- cgit v1.2.3