From 7af62a670c8b29050c3c1dcf0f4175ea18377de6 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sat, 10 Dec 2011 22:35:52 +0100 Subject: Document group specifications in all/0 and the group info function --- lib/common_test/doc/src/common_test_app.xml | 222 ++++++++++++++++++------- lib/common_test/doc/src/write_test_chapter.xml | 66 +++++++- 2 files changed, 222 insertions(+), 66 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index f58b2ab0a9..c7f6c7ce5c 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -69,12 +69,25 @@ - Module:all() -> TestCases | {skip,Reason} - Returns the list of all test cases in the module. + Module:all() -> Tests | {skip,Reason} + Returns the list of all test case groups and test cases + in the module. - TestCases = [atom() | {group,GroupName}] - Reason = term() + Tests = [TestCase | {group,GroupName} | + {group,GroupName,Properties} | + {group,GroupName,Properties,SubGroups}] + TestCase = atom() GroupName = atom() + Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | + default + SubGroups = [{GroupName,Properties} | + {GroupName,Properties,SubGroups}] + Shuffle = shuffle | {shuffle,Seed} + Seed = {integer(),integer(),integer()} + RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | + repeat_until_any_ok | repeat_until_any_fail + N = integer() | forever + Reason = term() @@ -85,9 +98,14 @@ This list also specifies the order the cases and groups will be executed by Common Test. A test case is represented by an atom, the name of the test case function. A test case group is - represented by a {group,GroupName} tuple, where - GroupName, an atom, is the name of the group (defined - with groups/0).

+ represented by a group tuple, where GroupName, + an atom, is the name of the group (defined in groups/0). + Execution properties for groups may also be specified, both + for a top level group and for any of its sub-groups. + Group execution properties specified here, will override + properties in the group definition (see groups/0). + (With value default, the group definition properties + will be used).

If {skip,Reason} is returned, all test cases in the module will be skipped, and the Reason will @@ -120,14 +138,16 @@

OPTIONAL

-

See Test case +

Function for defining test case groups. Please see + Test case groups in the User's Guide for details.

 Module:suite() -> [Info] - Test suite info function (providing default data for the suite). + Test suite info function (providing default data + for the suite). Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | @@ -160,11 +180,11 @@

This is the test suite info function. It is supposed to return a list of tagged tuples that specify various properties - regarding the execution of this test suite (common for all + related to the execution of this test suite (common for all test cases in the suite).

The timetrap tag sets the maximum time each - test case is allowed to take (including init_per_testcase/2 + test case is allowed to execute (including init_per_testcase/2 and end_per_testcase/2). If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. If a TimeFunc function is specified, @@ -172,9 +192,9 @@ TimeVal format.

The require tag specifies configuration variables - that are required by test cases in the suite. If the required - configuration variables are not found in any of the - configuration files, all test cases are skipped. For more + that are required by test cases (and/or configuration functions) + in the suite. If the required configuration variables are not found + in any of the configuration files, all test cases are skipped. For more information about the 'require' functionality, see the reference manual for the function ct:require/[1,2].

@@ -196,8 +216,9 @@ - Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | {skip_and_save,Reason,SaveConfig} - Test suite initialization. + Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | + {skip_and_save,Reason,SaveConfig} + Test suite initializations. Config = NewConfig = SaveConfig = [{Key,Value}] Key = atom() @@ -208,15 +229,15 @@

OPTIONAL

-

This function is called as the first function in the - suite. It typically contains initialization which is common for +

This configuration function is called as the first function in the + suite. It typically contains initializations which are common for all test cases in the suite, and which shall only be done - once. The Config parameter is the configuration + once. The Config parameter is the configuration data which can be modified here. Whatever is returned from this function is given as Config to all configuration functions and test cases in the suite. If {skip,Reason} - is returned, all test cases in the suite will be skipped and Reason - printed in the overview log for the suite.

+ is returned, all test cases in the suite will be skipped + and Reason printed in the overview log for the suite.

For information on save_config and skip_and_save, please see Dependencies @@ -224,29 +245,106 @@ - - Module:end_per_suite(Config) -> void() | {save_config,SaveConfig} - Test suite finalization. - - Config = SaveConfig = [{Key,Value}] - Key = atom() - Value = term() - + + Module:end_per_suite(Config) -> void() | + {save_config,SaveConfig} + Test suite finalization. + + Config = SaveConfig = [{Key,Value}] + Key = atom() + Value = term() + + + +

OPTIONAL

+

This function is called as the last test case in the + suite. It is meant to be used for cleaning up after + init_per_suite/1. + For information on save_config, please see + Dependencies + between Test Cases and Suites in the User's Guide.

+ + + + + Module:group(GroupName) -> [Info] + Test case group info function (providing default data + for a test case group, i.e. its test cases and sub-groups). + + Info = {timetrap,Time} | {require,Required} | + {require,Name,Required} | {userdata,UserData} | + {silent_connections,Conns} | {stylesheet,CSSFile} | + {ct_hooks, CTHs} + Time = TimeVal | TimeFunc + TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | + {hours,integer()} + TimeFunc = {Mod,Func,Args} | Fun + MilliSec = integer() + Mod = atom() + Func = atom() + Args = list() + Fun = fun() + Required = Key | {Key,SubKeys} + Key = atom() + SubKeys = SubKey | [SubKey] + SubKey = atom() + Name = atom() + UserData = term() + Conns = [atom()] + CSSFile = string() + CTHs = [CTHModule | {CTHModule, CTHInitArgs} | + {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + +

OPTIONAL

-

This function is called as the last test case in the - suite. It is meant to be used for cleaning up after init_per_suite/1. - For information on save_config, please see - Dependencies between - Test Cases and Suites in the User's Guide.

- +

This is the test case group info function. It is supposed to + return a list of tagged tuples that specify various properties + related to the execution of a test case group (i.e. its test cases + and sub-groups). Properties set by groups/1 override + properties with the same key that have been previously set by + suite/0.

+ +

The timetrap tag sets the maximum time each + test case is allowed to execute (including init_per_testcase/2 + and end_per_testcase/2). If the timetrap time is + exceeded, the test case fails with reason + timetrap_timeout. If a TimeFunc function is specified, + it will be called initially and must return a value on + TimeVal format.

+ +

The require tag specifies configuration variables + that are required by test cases (and/or configuration functions) + in the suite. If the required configuration variables are not found + in any of the configuration files, all test cases in this group are skipped. + For more information about the 'require' functionality, see the + reference manual for the function + ct:require/[1,2].

+ +

With userdata, it is possible for the user to + specify arbitrary test case group related information which can be + read by calling ct:userdata/2.

+ +

The ct_hooks tag specifies which + Common Test Hooks + are to be run together with this suite.

+ +

Other tuples than the ones defined will simply be ignored.

+ +

For more information about the test case group info function, + see Test + case group info function in the User's Guide.

+ - Module:init_per_group(GroupName, Config) -> NewConfig | {skip,Reason} - Test case group initialization. + Module:init_per_group(GroupName, Config) -> NewConfig | + {skip,Reason} + Test case group initializations. GroupName = atom() Config = NewConfig = [{Key,Value}] @@ -258,16 +356,16 @@

OPTIONAL

-

This function is called before execution of a test case group. - It typically contains initialization which is common for - all test cases in the group, and which shall only be performed - once. GroupName is the name of the group, as specified in - the group definition (see groups/0). The Config - parameter is the configuration which can be modified here. - Whatever is returned from this function is given as Config - to all test cases in the group. If {skip,Reason} is returned, - all test cases in the group will be skipped and Reason printed - in the overview log for the group.

+

This configuration function is called before execution of a + test case group. It typically contains initializations which are + common for all test cases and sub-groups in the group, and which + shall only be performed once. GroupName is the name of the + group, as specified in the group definition (see groups/0). The + Config parameter is the configuration data which can be modified + here. The return value of this function is given as Config + to all test cases and sub-groups in the group. If {skip,Reason} + is returned, all test cases in the group will be skipped and + Reason printed in the overview log for the group.

For information about test case groups, please see Test case @@ -276,7 +374,8 @@ - Module:end_per_group(GroupName, Config) -> void() | {return_group_result,Status} + Module:end_per_group(GroupName, Config) -> void() | + {return_group_result,Status} Test case group finalization. GroupName = atom() @@ -305,7 +404,7 @@ Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail,Reason} | {skip,Reason} - Test case initialization. + Test case initializations. TestCase = atom() Config = NewConfig = [{Key,Value}] @@ -385,10 +484,13 @@

This is the test case info function. It is supposed to return a list of tagged tuples that specify various properties - regarding the execution of this particular test case.

+ related to the execution of this particular test case. + Properties set by Testcase/0 override + properties that have been previously set for the test case + by group/1 or suite/0.

The timetrap tag sets the maximum time the - test case is allowed to take. If the timetrap time is + test case is allowed to execute. If the timetrap time is exceeded, the test case fails with reason timetrap_timeout. init_per_testcase/2 and end_per_testcase/2 are included in the @@ -397,16 +499,16 @@ and must return a value on TimeVal format.

The require tag specifies configuration variables - that are required by the test case. If the required - configuration variables are not found in any of the + that are required by the test case (and/or init/end_per_testcase/2). + If the required configuration variables are not found in any of the configuration files, the test case is skipped. For more information about the 'require' functionality, see the reference manual for the function ct:require/[1,2].

If timetrap and/or require is not set, the - default values specified in the suite/0 return list - will be used.

+ default values specified by suite/0 (or + group/1) will be used.

With userdata, it is possible for the user to specify arbitrary test case related information which can be @@ -438,12 +540,12 @@

This is the implementation of a test case. Here you must call the functions you want to test, and do whatever you need to check the result. If something fails, make sure the - function causes a runtime error, or call ct:fail/[0,1] - (which also causes the test case process to crash).

+ function causes a runtime error, or call ct:fail/1/2 + (which also causes the test case process to terminate).

-

Elements from the Config parameter can be read - with the ?config macro. The config - macro is defined in ct.hrl

+

Elements from the Config list can e.g. be read + with proplists:get_value/2 (or the macro ?config + defined in ct.hrl).

You can return {skip,Reason} if you decide not to run the test case after all. Reason will then be diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index e35888e68f..b8487ea2d4 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -68,7 +68,7 @@

Each test suite module must export the function all/0 which returns the list of all test case groups and test cases - in that module. + to be executed in that module.

@@ -450,11 +450,65 @@ 
       all() -> [testcase1, {group,group1}, testcase2, {group,group2}].
-

Properties may be combined so that e.g. if shuffle, - repeat_until_any_fail and sequence are all specified, the test - cases in the group will be executed repeatedly and in random order until - a test case fails, when execution is immediately stopped and the rest of - the cases skipped.

+

It is also possible to specify execution properties with a group + tuple in all/0: {group,GroupName,Properties}. These + properties will override those specified in the group definition (see + groups/0 above). This way, it's possible to run the same set of tests, + but with different properties, without having to make copies of the group + definition in question.

+ +

If a group contains sub-groups, the execution properties for these may + also be specified in the group tuple: + {group,GroupName,Properties,SubGroups}, where SubGroups + is a list of tuples, {GroupName,Properties}, or + {GroupName,Properties,SubGroups}, representing the sub-groups. + Any sub-groups defined in group/0 for a group, that are not specified + in the SubGroups list, will simply execute with their pre-defined + properties.

+ +

Example:

+ 
+      groups() -> {tests1, [], [{tests2, [], [t2a,t2b]},
+                                {tests3, [], [t31,t3b]}]}.
+

To execute group 'tests1' twice with different properties for 'tests2' + each time:

+ 
+      all() ->
+         [{group, tests1, default, [{tests2, [parallel]}]},
+          {group, tests1, default, [{tests2, [shuffle,{repeat,10}]}]}].
+

Note that this is equivalent to this specification:

+ 
+      all() ->
+         [{group, tests1, default, [{tests2, [parallel]},
+                                    {tests3, default}]},
+          {group, tests1, default, [{tests2, [shuffle,{repeat,10}]},
+                                    {tests3, default}]}].
+

The value default states that the pre-defined properties + should be used.

+

Here's an example of how to override properties in a scenario + with deeply nested groups:

+ 
+      groups() ->
+         [{tests1, [], [{group, tests2}]},
+          {tests2, [], [{group, tests3}]},
+          {tests3, [{repeat,2}], [t3a,t3b,t3c]}].
+
+      all() ->
+         [{group, tests1, default, 
+           [{tests2, default,
+             [{tests3, [parallel,{repeat,100}]}]}]}].
+ +

The syntax described above may also be used in Test Specifications + in order to change properties of groups at the time of execution, + without even having to edit the test suite (please see the + Test + Specifications chapter for more info).

+ +

As illustrated above, properties may be combined. If e.g. + shuffle, repeat_until_any_fail and sequence + are all specified, the test cases in the group will be executed + repeatedly, and in random order, until a test case fails. Then + execution is immediately stopped and the rest of the cases skipped.

Before execution of a group begins, the configuration function init_per_group(GroupName, Config) is called (the function is -- cgit v1.2.3 From c96a29d78a8ced79557582c631a9082705d772a3 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 11 Dec 2011 00:35:21 +0100 Subject: Document changes in test specifications (groups) and style sheets --- lib/common_test/doc/src/run_test_chapter.xml | 49 +++++++++++----------------- 1 file changed, 19 insertions(+), 30 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 57059f0ba2..06d1ba8b03 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -412,6 +412,16 @@ the way from init_per_suite down to the test cases in the sub group).

+

With the GroupSpec element (below) it's possible to specify + group execution properties that will override those specified in the + group definition (i.e. in groups/0). Execution properties for + sub-groups may be overridden as well. This feature makes it possible to + change properties of groups at the time of execution, + without even having to edit the test suite. More detailed documentation, + and examples, can be found in the + + Test case groups chapter.

+

Below is the test specification syntax. Test specifications can be used to run tests both in a single test host environment and in a distributed Common Test environment (Large Scale @@ -475,8 +485,8 @@ {groups, DirRef, Suite, Groups}. {groups, NodeRefsDirRef, Suite, Groups}. - {groups, DirRef, Suite, Group, {cases,Cases}}. - {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}. + {groups, DirRef, Suite, GroupSpec, {cases,Cases}}. + {groups, NodeRefsDirRef, Suite, GroupSpec, {cases,Cases}}. {cases, DirRef, Suite, Cases}. {cases, NodeRefs, DirRef, Suite, Cases}. @@ -510,7 +520,8 @@ DirRef = DirAlias | Dir Suites = atom() | [atom()] | all Suite = atom() - Groups = atom() | [atom()] | all + Groups = GroupSpec | [GroupSpec] | all + GroupSpec = Group | {Group,Properties} | {Group,Properties,GroupSpec} Group = atom() Cases = atom() | [atom()] | all Comment = string() | "" @@ -708,22 +719,16 @@ look like this:

-<style>
-  div.ct_internal { background:lightgrey; color:black }
-  div.default     { background:lightgreen; color:black }
-  div.sys_config  { background:blue; color:white }
-  div.sys_state   { background:yellow; color:black }
-  div.error       { background:red; color:white }
-</style>
-
+ div.sys_config { background:blue; color:white } + div.sys_state { background:yellow; color:black } + div.error { background:red; color:white }

To install the CSS file (Common Test inlines the definition in the HTML code), the name may be provided when executing ct_run. Example:

-	  $ ct_run -dir $TEST/prog -stylesheet $TEST/styles/test_categories.css
-
+ $ ct_run -dir $TEST/prog -stylesheet $TEST/styles/test_categories.css

Categories in a CSS file installed with the -stylesheet flag are on a global test level in the sense that they can be used in any @@ -744,8 +749,7 @@ ct:log(sys_state, "Connections: ~p", [ConnectionInfo]), ... ct:pal(error, "Error ~p detected! Info: ~p", [SomeFault,ErrorInfo]), - ct:fail(SomeFault). - + ct:fail(SomeFault).

If the style sheet is installed as in this example, the categories are private to the suite in question. They can be used by all test cases in the @@ -769,21 +773,6 @@

The Category argument in the example above may have the value (atom) sys_config (white on blue), sys_state (black on yellow) or error (white on red).

- -

If the Category argument is not specified, Common Test will - use the CSS selector div.default for the - printout. For this reason a user supplied style sheet must - include this selector. Also the selector - div.ct_internal must be included. Hence a minimal - user style sheet should look like this (which is also the - default style sheet Common Test uses if no user CSS file is - provided):

- 
-	  <style>
-	  div.ct_internal { background:lightgrey; color:black }
-	  div.default     { background:lightgreen; color:black }
-	  </style>
-
-- cgit v1.2.3 From 5f93ed61f48b5d90d1217215d16a8ca61120dd66 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 11 Dec 2011 01:07:11 +0100 Subject: Document the group info function --- lib/common_test/doc/src/write_test_chapter.xml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index b8487ea2d4..9f2cf73ef2 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -694,6 +694,28 @@ sub-group also has a shuffle property).

+
+ + Group info function + +

The test case group info function, group(GroupName), + serves the same purpose as the suite- and test case info + functions previously described in this chapter. The scope for + the group info, however, is all test cases and sub-groups in the + group in question (GroupName).

+

Example:

+ 
+	group(connection_tests) ->
+	   [{require,login_data},
+	    {timetrap,1000}].
+ +

The group info properties override those set with the + suite info function, and may in turn be overridden by test + case info properties. Please see the suite- and test case info + functions above for a list of valid info properties and more + general information.

+
+
Data and Private Directories -- cgit v1.2.3 From f27bd68aa4089c7c17add9cdb90591de932060a6 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 11 Dec 2011 20:42:59 +0100 Subject: Document info functions for init/end_per_suite/group --- lib/common_test/doc/src/write_test_chapter.xml | 37 +++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 9f2cf73ef2..c0ec26ddcc 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -369,10 +369,12 @@ Test suite info function

The suite/0 function can be used in a test suite - module to set the default values for the timetrap and - require tags. If a test case info function also specifies - any of these tags, the default value is overruled. See above for - more information. + module to e.g. set a default timetrap value and to + require external configuration data. If a test case-, or + group info function also specifies any of the info tags, it + overrides the default values set by suite/0. See the test + case info function above, and group info function below, for more + details.

Other options that may be specified with the suite info list are:

@@ -711,11 +713,34 @@

The group info properties override those set with the suite info function, and may in turn be overridden by test - case info properties. Please see the suite- and test case info - functions above for a list of valid info properties and more + case info properties. Please see the test case info + function above for a list of valid info properties and more general information.

+
+ Info functions for init- and end-configuration +

It is possible to use info functions also for the init_per_suite, + end_per_suite, init_per_group, and end_per_group + functions, and it works the same way as with info functions + for test cases (see above). This is useful e.g. for setting + timetraps and requiring external configuration data relevant + only for the configuration function in question (without + affecting properties set for groups and test cases in the suite).

+ +

The info function init/end_per_suite() is called for + init/end_per_suite(Config), and info function + init/end_per_group(GroupName) is called for + init/end_per_group(GroupName,Config). Info functions + can not be used with init/end_per_testcase(TestCase, Config), + however, since these configuration functions execute on the test case process + and will use the same properties as the test case (i.e. the properties + set by the test case info function, TestCase()). Please see the test case + info function above for a list of valid info properties and more + general information. +

+
+
Data and Private Directories -- cgit v1.2.3 From 76a362c649d78b5790a67df62b757d6ce5403ef3 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 11 Dec 2011 21:06:37 +0100 Subject: Document new CSS look and the basic_html flag --- lib/common_test/doc/src/run_test_chapter.xml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 06d1ba8b03..a699cf425e 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -658,7 +658,7 @@

The minor log file contain full details of every single test case, each one in a separate file. This way the files should be easy to compare with previous test runs, even if the set of - test cases change. If SASL is running those logs will also be + test cases change. If SASL is running, those logs will also be printed there by the cth_log_redirect built-in hook. @@ -704,8 +704,15 @@

HTML Style Sheets -

Common Test includes the optional feature to use - HTML style sheets (CSS) for customizing user printouts. The +

Common Test uses a CSS file to control the look of the HTML + files generated during test runs. If, for some reason, the + log files are not displayed correctly in the HTML browser of your + choice, or you prefer the "pre Common Test v1.6 look" + of the log files (i.e. not using CSS), use the start flag/option + basic_html to revert to the old style.

+ +

Common Test includes an optional feature to allow + user HTML style sheets for customizing printouts. The functions in ct that print to a test case HTML log file (log/3 and pal/3) accept Category as first argument. With this argument it's possible to -- cgit v1.2.3 From 46ef0536a9e3442c3a54237b59b238e6b6f7a90a Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 11 Dec 2011 22:27:34 +0100 Subject: Document the tc_logfile data --- lib/common_test/doc/src/event_handler_chapter.xml | 10 ++++++++++ lib/common_test/doc/src/run_test_chapter.xml | 23 +++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index b41b233ce6..a5886b9687 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -174,6 +174,16 @@ are also given.

+ #event{name = tc_logfile, data = {{Suite,Func},LogFileName}} +

Suite = atom(), name of the test suite.

+

Func = atom(), name of test case or configuration function.

+

LogFileName = string(), full name of test case log file.

+

This event is sent at the start of each test case (and configuration function + except init/end_per_testcase) and carries information about the + full name (i.e. the file name including the absolute directory path) of + the current test case log file. +

+ #event{name = tc_done, data = {Suite,FuncOrGroup,Result}}

Suite = atom(), name of the suite.

diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index a699cf425e..848f278fa6 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -655,14 +655,25 @@ to each individual test case log file for quick viewing with an HTML browser.

-

The minor log file contain full details of every single test - case, each one in a separate file. This way the files should - be easy to compare with previous test runs, even if the set of - test cases change. If SASL is running, those logs will also be - printed there by the +

The minor log files contain full details of every single test + case, each one in a separate file. This way, it should be + straightforward to compare the latest results to that of previous + test runs, even if the set of test cases changes. If SASL is running, + its logs will also be printed to the current minor log file by the cth_log_redirect built-in hook. -

+

+ +

The full name of the minor log file (i.e. the name of the file + including the absolute directory path) can be read during execution + of the test case. It comes as value in the tuple + {tc_logfile,LogFileName} in the Config list (which means it + can also be read by a pre- or post Common Test hook function). Also, + at the start of a test case, this data is sent with an event + to any installed event handler. Please see the + Event Handling + chapter for details. +

Which information goes where is user configurable via the test server controller. Three threshold values determine what -- cgit v1.2.3 From e7fc8ebdf5fcf82c581026d7c52e23a53b05ba01 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Mon, 12 Dec 2011 03:10:51 +0100 Subject: Update basics chapter and introduce getting started chapter --- lib/common_test/doc/src/Makefile | 6 +- lib/common_test/doc/src/basics_chapter.xml | 75 ++++--- lib/common_test/doc/src/config.gif | Bin 0 -> 4963 bytes .../doc/src/getting_started_chapter.xml | 230 +++++++++++++++++++++ lib/common_test/doc/src/html_logs.gif | Bin 0 -> 10726 bytes lib/common_test/doc/src/part.xml | 1 + lib/common_test/doc/src/tc_execution.gif | Bin 0 -> 9561 bytes 7 files changed, 280 insertions(+), 32 deletions(-) create mode 100644 lib/common_test/doc/src/config.gif create mode 100644 lib/common_test/doc/src/getting_started_chapter.xml create mode 100644 lib/common_test/doc/src/html_logs.gif create mode 100644 lib/common_test/doc/src/tc_execution.gif (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index 964f7c76c1..d9651f13b0 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -61,6 +61,7 @@ XML_PART_FILES = part.xml XML_CHAPTER_FILES = \ basics_chapter.xml \ + getting_started_chapter.xml \ install_chapter.xml \ write_test_chapter.xml \ test_structure_chapter.xml \ @@ -80,7 +81,10 @@ MAKE_EDOC = make_edoc BOOK_FILES = book.xml -GIF_FILES = +GIF_FILES = \ + tc_execution.gif \ + config.gif \ + html_logs.gif INSTALL_NOTES = ../../notes.html diff --git a/lib/common_test/doc/src/basics_chapter.xml b/lib/common_test/doc/src/basics_chapter.xml index c1bb365b1f..20141d2561 100644 --- a/lib/common_test/doc/src/basics_chapter.xml +++ b/lib/common_test/doc/src/basics_chapter.xml @@ -28,54 +28,65 @@ basics_chapter.xml - +

Introduction -

- The Common Test framework (CT) is a tool which can support - implementation and automated execution of test cases towards different - types of target systems. The framework is based on the OTP Test - Server. Test cases can be run individually or in batches. Common - Test also features a distributed testing mode with central - control and logging. This feature makes it possible to test - multiple systems independently in one common session. This - can be very useful e.g. for running automated large-scale regression - tests. +

The Common Test framework (CT) is a tool which supports + implementation and automated execution of test cases towards arbitrary + types of target systems. The CT framework is based on the OTP Test + Server and it's the main tool being used in all testing- and verification + activities that are part of Erlang/OTP system development- and maintenance. +

+ +

Test cases can be executed individually or in batches. Common Test + also features a distributed testing mode with central control and logging + (a feature that makes it possible to test multiple systems independently in + one common session, useful e.g. for running automated large-scale regression + tests).

The SUT (System Under Test) may consist of one or several target - nodes. CT contains a generic test server which together with - other test utilities is used to perform test case execution. - It is possible to start the tests from the CT GUI or from an OS- or - Erlang shell prompt. Test suites are files (Erlang + nodes. CT contains a generic test server which, together with + other test utilities, is used to perform test case execution. + It is possible to start the tests from a GUI or from the OS- or + Erlang shell. Test suites are files (Erlang modules) that contain the test cases (Erlang functions) to be executed. Support modules provide functions that the test cases utilize in order to carry out the tests.

-

- The main idea is that CT based test programs connect to - the target system(s) via standard O&M interfaces. CT - provides implementations and wrappers of some of these O&M - interfaces and will be extended with more interfaces later. - There are a number of target independent interfaces - supported in CT such as Generic Telnet, FTP etc. which can be - specialized or used directly for controlling instruments, - traffic generators etc.

+

In a black-box testing scenario, CT based test programs connect to + the target system(s) via standard O&M and CLI protocols. CT + provides implementations of, and wrapper interfaces to, some of these + protocols (most of which exist as stand-alone components and + applications in OTP). The wrappers simplify configuration and add + verbosity for logging purposes. CT will be continously extended with + useful support modules. (Note however that it's + a straightforward task to use any arbitrary Erlang/OTP component + for testing purposes with Common Test, without needing a CT wrapper + for it. It's as simple as calling Erlang functions). There + are a number of target independent interfaces supported in CT, such as + Generic Telnet, FTP, etc, which can be specialized or used + directly for controlling instruments, traffic load generators, etc. +

Common Test is also a very useful tool for white-box testing Erlang - code since the test programs can call Erlang API functions directly. - For black-box testing Erlang software, Erlang RPC as well as - standard O&M interfaces can be used. + code (e.g. module testing), since the test programs can call exported Erlang + functions directly and there's very little overhead required for + implementing basic test suites and executing simple tests. For black-box + testing Erlang software, Erlang RPC as well as standard O&M interfaces + can for example be used.

A test case can handle several connections towards one or several target systems, instruments and traffic generators in parallel in order to perform the necessary actions for a test. The handling of many connections in parallel is one of - the major strengths of Common Test! + the major strengths of Common Test (thanks to the efficient + support for concurrency in the Erlang runtime system - which CT users + can take great advantage of!).

@@ -186,7 +197,7 @@ all() - Returns a list of all test cases in the suite. (Mandatory) + Returns a list of all test cases and groups in the suite. (Mandatory) suite() Info function used to return properties for the suite. (Optional) groups() @@ -197,12 +208,14 @@ end_per_suite(Config) Suite level configuration function, executed after the last test case. (Optional) + group(GroupName) + Info function used to return properties for a test case group. (Optional) init_per_group(GroupName, Config) Configuration function for a group, executed before the first - test case. (Mandatory if groups are defined) + test case. (Optional) end_per_group(GroupName, Config) Configuration function for a group, executed after the last - test case. (Mandatory if groups are defined) + test case. (Optional) init_per_testcase(TestCase, Config) Configuration function for a testcase, executed before each test case. (Optional) diff --git a/lib/common_test/doc/src/config.gif b/lib/common_test/doc/src/config.gif new file mode 100644 index 0000000000..ac8006c4fb Binary files /dev/null and b/lib/common_test/doc/src/config.gif differ diff --git a/lib/common_test/doc/src/getting_started_chapter.xml b/lib/common_test/doc/src/getting_started_chapter.xml new file mode 100644 index 0000000000..7de0912036 --- /dev/null +++ b/lib/common_test/doc/src/getting_started_chapter.xml @@ -0,0 +1,230 @@ + + + + +
+ + 20072010 + Ericsson AB. All Rights Reserved. + + + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + + + Getting Started + Peter Andersson + + 2011-12-12 + + getting_started_chapter.xml +
+ +
+ Are you new around here? +

+ The purpose of this short chapter is to, with a "learning by example" + approach, give the newcomer a chance to get started quickly writing and + executing some first simple tests. The chapter will introduce some of the + basics, but leave most explanations and details for the later + chapters in this User's Guide. Hopefully though, after this chapter, you + will be inspired and unintimidated enough to go on and get into the + nitty-gritty that follows in this rather heavy User's Guide! If you're + not much into "learning by example" and prefer to get into more technical + detail right away, go ahead and skip to the next chapter. Again, the basics + presented here will be covered in detail in later chapters. +

+

+ This chapter also tries to demonstrate how dead simple it actually is + to write a very basic (yet for many module testing purposes, often sufficiently + complex) test suite, and execute its test cases. This is not necessarily + obvious when you read the rest of the chapters in the User's Guide. +

+

+ A quick note before we start: In order to understand what's discussed and + examplified here, it is recommended that you first read through the + opening Common Test Basics + chapter. +

+
+ +
+ Test case execution +

Execution of test cases is handled this way:

+ +

+ + + Successful vs unsuccessful test case execution. + + +

+ +

For each test case that Common Test is told to execute, it spawns a + dedicated process on which the test case function in question starts + running. (In parallel to the test case process, an idle waiting timer + process is started which is linked to the test case process. If the timer + process runs out of waiting time, it sends an exit signal to terminate + the test case process and this is what's called a timetrap). +

+

In scenario 1, the test case process terminates normally after case A has + finished executing its test code without detecting any errors. The test + case function simply returns a value and Common Test logs the test case as + successful. +

+

In scenario 2, an error is detected during test case execution + which causes the test case B function to generate an exception. + This causes the test case process to exit with reason + other than normal, and as a result, Common Test will log this as an + unsuccessful test case. +

+

As you can understand from the illustration above, Common Test requires + that a test case generates a runtime error to indicate failure (e.g. + by causing a bad match error or by calling exit/1, preferrably + through the ct:fail/1/2 help function). A succesful execution is + indicated by means of a normal return from the test case function. +

+
+ +
+ A simple test suite +

As you've seen in the basics chapter, the test suite module implements + callback functions (mandatory or optional) for various purposes, e.g: + + Init/end configuration function for the test suite + Init/end configuration function for a test case + Init/end configuration function for a test case group + Test cases + + The configuration functions are optional and if you don't need them for + your test, a test suite with one simple test case could look like this: +

+ 
+      -module(my1st_SUITE).
+      -compile(export_all).
+
+      all() ->
+          [mod_exists].
+
+      mod_exists(_) ->
+          {module,mymod} = code:load_file(mymod).
+

+ In this example we check that the mymod module exists (i.e. can be + successfully loaded by the code server). If the operation fails, we will + get a bad match error which terminates the test case. +

+
+ +
+ A test suite with configuration functions +

+ If we need to perform configuration operations in order to run our test, we + implement configuration functions in our suite. The result from a + configuration function is configuration data, or simply Config. + This is a list of key-value tuples which get passed from the configuration + function to the test cases (possibly through configuration functions on + "lower level"). The data flow looks like this: +

+ +

+ + + Config data flow in the suite. + + +

+ +

+ Here's an example of a test suite which uses configuration functions + to open and close a log file for the test cases (an operation that would + be unnecessary and irrelevant to perform by each test case): +

+ 
+      -module(check_log_SUITE).
+      -export([all/0, init_per_suite/1, end_per_suite/1]).
+      -export([check_restart_result/1, check_no_errors/1]).
+      
+      -define(value(Key,Config), proplists:get_value(Key,Config)).
+
+      all() -> [check_restart_result, check_no_errors].
+
+      init_per_suite(InitConfigData) ->
+          [{logref,open_log()} | InitConfigData].
+
+      end_per_suite(ConfigData) ->
+          close_log(?value(logref, ConfigData)).
+
+      check_restart_result(ConfigData) ->
+          TestData = read_log(restart, ?value(logref, ConfigData)),
+          {match,_Line} = search_for("restart successful", TestData).
+      
+      check_no_errors(ConfigData) ->
+          TestData = read_log(all, ?value(logref, ConfigData)),
+          case search_for("error", TestData) of
+              {match,Line} -> ct:fail({error_found_in_log,Line});
+              nomatch -> ok
+          end.
+

+ In this example we have test cases that verify, by parsing a + log file, that our SUT has performed a successful restart and + that no unexpected errors have been printed. +

+ +

To execute the test cases in the test suite above, we could type this on + the Unix/Linux command line (assuming for this example that the suite module + is in the current working directory): +

+ 
+      $ ct_run -dir .
+

or

+ 
+    $ ct_run -suite check_log_SUITE
+ +

If we want to use the Erlang shell to run our test, we could evaluate this call: +

+ 
+      1> ct:run_test([{dir, "."}]).
+

or

+ 
+      1> ct:run_test([{suite, "check_log_SUITE"}]).
+

+ The result from running our test is printed in log files on HTML format + (stored in unique log directories on different level). This illustration + shows the log file structure: +

+ +

+ + + HTML log file structure. + + +

+
+ +
+ What happens next? +

+ You will find detailed information about the basics introduced here in this + chapter in the following chapters in the User's Guide, as well as + presentations of many more useful features. Have fun! +

+
+ + + + + + + + + diff --git a/lib/common_test/doc/src/html_logs.gif b/lib/common_test/doc/src/html_logs.gif new file mode 100644 index 0000000000..3a3fd86bde Binary files /dev/null and b/lib/common_test/doc/src/html_logs.gif differ diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 3284bcadaa..a74185221d 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -65,6 +65,7 @@ + diff --git a/lib/common_test/doc/src/tc_execution.gif b/lib/common_test/doc/src/tc_execution.gif new file mode 100644 index 0000000000..7c89d7be57 Binary files /dev/null and b/lib/common_test/doc/src/tc_execution.gif differ -- cgit v1.2.3