From 6288b704f2ee0699407f645536ad69e0dd07756d Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Thu, 2 Dec 2010 15:27:10 +0100 Subject: Started work on documenting suite callbacks, this is a partial commit --- lib/common_test/doc/src/Makefile | 2 +- lib/common_test/doc/src/ct_suite_callbacks.xml | 94 ++++++++++++++++++++++++++ lib/common_test/doc/src/ref_man.xml | 1 + 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 lib/common_test/doc/src/ct_suite_callbacks.xml (limited to 'lib/common_test/doc') diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index 1a767a8197..be066143e0 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -52,7 +52,7 @@ CT_XML_FILES = $(CT_MODULES:=.xml) XML_APPLICATION_FILES = ref_man.xml XML_REF1_FILES = ct_run.xml -XML_REF3_FILES = $(CT_XML_FILES) +XML_REF3_FILES = $(CT_XML_FILES) ct_suite_callbacks.xml XML_REF6_FILES = common_test_app.xml XML_PART_FILES = part.xml diff --git a/lib/common_test/doc/src/ct_suite_callbacks.xml b/lib/common_test/doc/src/ct_suite_callbacks.xml new file mode 100644 index 0000000000..28a496a47b --- /dev/null +++ b/lib/common_test/doc/src/ct_suite_callbacks.xml @@ -0,0 +1,94 @@ + + + + + +
+ + 20102010 + Ericsson AB. All Rights Reserved. + + + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + + + Suite Callbacks + Lukas Larsson + Lukas Larsson + + + + 2010-12-02 + PA1 + suite_callback.sgml +
+ ct_suite_callback + A callback interface on top of common test + + + +

The Suite Callback framework allows extensions of the default + behaviour of Common Test by means of callbacks before and after all + test suite calls.

+ +

In brief, Suite Callbacks allows you to:

+ + + Manipulating the runtime config before each suite configuration calls + Manipulating the return of all suite configuration calls + + +

The following sections describe the mandatory and optional suite callback + functions Common Test will call during test execution. For more details + see Suite Callbacks in the User's Guide.

+ + + +
+ CALLBACK FUNCTIONS +

The following functions define the callback interface + for a suite callback.

+
+ + + + Module:init(Opts) -> {Id,State} + asdas + + Opts = term() + Id = term() + GroupName = term() + + + +

MANDATORY

+ +

+ +

If {skip,Reason} is returned, all test cases + in the module will be skipped, and the Reason will + be printed on the HTML result page.

+ +

For details on groups, see + Test case + groups in the User's Guide.

+ + + + + + + + + + diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index d5985bb021..100c0fe5d7 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -76,6 +76,7 @@ + -- cgit v1.2.3 From 15e8dd20b5ba2c82e683e87254f18c9af3625481 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 9 Feb 2011 16:54:20 +0100 Subject: Add documentation for SCBs --- lib/common_test/doc/src/Makefile | 1 + lib/common_test/doc/src/common_test_app.xml | 10 +- lib/common_test/doc/src/ct_suite_callbacks.xml | 503 ++++++++++++++++++++- lib/common_test/doc/src/event_handler_chapter.xml | 11 +- lib/common_test/doc/src/part.xml | 1 + lib/common_test/doc/src/run_test_chapter.xml | 11 +- .../doc/src/suite_callbacks_chapter.xml | 394 ++++++++++++++++ lib/common_test/doc/src/write_test_chapter.xml | 1 + 8 files changed, 905 insertions(+), 27 deletions(-) create mode 100644 lib/common_test/doc/src/suite_callbacks_chapter.xml (limited to 'lib/common_test/doc') diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index be066143e0..a3f740852c 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -71,6 +71,7 @@ XML_CHAPTER_FILES = \ cover_chapter.xml \ ct_master_chapter.xml \ event_handler_chapter.xml \ + suite_callbacks_chapter.xml \ dependencies_chapter.xml \ notes.xml \ notes_history.xml diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index e30eef2488..a735dc85a8 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -131,7 +131,8 @@ Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | - {silent_connections,Conns} | {stylesheet,CSSFile} + {silent_connections,Conns} | {stylesheet,CSSFile} | + {suite_callbacks, SCBs} Time = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()} MilliSec = integer() @@ -143,6 +144,9 @@ UserData = term() Conns = [atom()] CSSFile = string() + SCBs = [SCBModule | {SCBModule, SCBInitArgs}] + SCBModule = atom() + SCBInitArgs = term() @@ -170,6 +174,10 @@

With userdata, it is possible for the user to specify arbitrary test suite related information which can be read by calling ct:userdata/2.

+ +

The suite_callbacks tag specifies which + Suite Callbacks + are to be run together with this suite.

Other tuples than the ones defined will simply be ignored.

diff --git a/lib/common_test/doc/src/ct_suite_callbacks.xml b/lib/common_test/doc/src/ct_suite_callbacks.xml index 28a496a47b..cea5804825 100644 --- a/lib/common_test/doc/src/ct_suite_callbacks.xml +++ b/lib/common_test/doc/src/ct_suite_callbacks.xml @@ -1,4 +1,4 @@ - + @@ -33,24 +33,43 @@ suite_callback.sgml ct_suite_callback - A callback interface on top of common test + A callback interface on top of Common Test -

The Suite Callback framework allows extensions of the default - behaviour of Common Test by means of callbacks before and after all - test suite calls.

+

This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.

 + +

The Suite Callback (henceforth called SCB) framework allows + extensions of the default behaviour of Common Test by means of callbacks + before and after all test suite calls. It is meant for advanced users of + Common Test which want to abstract out behaviour which is common to + multiple test suites.

In brief, Suite Callbacks allows you to:

- Manipulating the runtime config before each suite configuration calls - Manipulating the return of all suite configuration calls + Manipulate the runtime config before each suite + configuration calls + Manipulate the return of all suite configuration calls and in + extension the result of the test themselves. -

The following sections describe the mandatory and optional suite callback - functions Common Test will call during test execution. For more details - see Suite Callbacks in the User's Guide.

+

The following sections describe the mandatory and optional SCB + functions Common Test will call during test execution. For more details + see Suite Callbacks in + the User's Guide.

+ +

For information about how to add a SCB to your suite see + Installing an SCB + in the User's Guide.

+ +

See the + Example SCB + in the User's Guide for a minimal example of an SCB.

@@ -62,31 +81,469 @@ - Module:init(Opts) -> {Id,State} - asdas + Module:init(Id, Opts) -> State + Initiates the Suite Callback + Id = reference() | term() Opts = term() - Id = term() - GroupName = term() + State = term()

MANDATORY

+ +

Always called before any other callback function. + Use this to initiate any common state. + It should return a state for this SCB.

+ +

Id is the return value of + id/1, or a reference + if id/1 is not implemented. +

+ +

For details about when init is called see + scope + in the User's Guide.

+ + + + + + Module:pre_init_per_suite(SuiteName, Config, SCBState) -> + Result + Called before init_per_suite + + SuiteName = atom() + Config = NewConfig = [{Key,Value}] + SCBState = NewSCBState = term() + Result = {Return, NewSCBState} + Return = NewConfig | SkipOrFail + SkipOrFail = {fail, Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + init_per_suite if it exists. + It typically contains initialization/logging which needs to be done + before init_per_suite is called. + If {skip,Reason} or {fail,Reason} is returned, + init_per_suite and all test cases of the suite will be skipped and + Reason printed in the overview log of the suite.

-

+

SuiteName is the name of the suite to be run.

+ +

Config is the original config list of the test suite.

+ +

SCBState is the current internal state of the SCB.

+ +

Return is the result of the init_per_suite function. + If it is {skip,Reason} or {fail,Reason} + init_per_suite + will never be called, instead the initiation is considered + to be skipped/failed respectively. If a NewConfig list + is returned, + init_per_suite will be called with that NewConfig list. + See + Manipulating tests in the User's Guide for more details.

+ + +

Note that this function is only called if the SCB has been added + before init_per_suite is run, see + SCB Scoping + in the User's Guide for details.

+ + + + + Module:post_init_per_suite(SuiteName, Config, Return, SCBState) -> + Result + Called after init_per_suite + + SuiteName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail, Reason} | {skip, Reason} | term() + SCBState = NewSCBState = term() + Result = {NewReturn, NewSCBState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

Return is what + init_per_suite + returned, i.e. {fail,Reason}, {skip,Reason}, a Config + list or a term describing how + init_per_suite + failed.

+ +

NewReturn is the possibly modified return value of + init_per_suite + . It is here possible to recover from a failure in + init_per_suite + by returning the ConfigList with the tc_status + element removed.

+ +

SCBState is the current internal state of the SCB.

+ +

This function is called after + + init_per_suite if it exists.

+ +

Note that this function is only called if the SCB has been added + before or in init_per_suite, see + SCB Scoping + in the User's Guide for details.

+ + + + + Module:pre_init_per_group(GroupName, Config, SCBState) -> + Result + Called before init_per_group + + GroupName = atom() + Config = NewConfig = [{Key,Value}] + SCBState = NewSCBState = term() + Result = {NewConfig | SkipOrFail, NewSCBState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

-

If {skip,Reason} is returned, all test cases - in the module will be skipped, and the Reason will - be printed on the HTML result page.

+

This function is called before + + init_per_group if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + init_per_group instead.

+ + + + + Module:post_init_per_group(GroupName, Config, Return, SCBState) -> + Result + Called after init_per_group + + GroupName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + SCBState = NewSCBState = term() + Result = {NewReturn, NewSCBState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

-

For details on groups, see - Test case - groups in the User's Guide.

+

This function is called after + + init_per_group if it exists. It behaves the same way as + + post_init_per_suite, but for the + + init_per_group instead.

+ + + + + Module:pre_init_per_testcase(TestcaseName, Config, SCBState) -> + Result + Called before init_per_testcase + + TestcaseName = atom() + Config = NewConfig = [{Key,Value}] + SCBState = NewSCBState = term() + Result = {NewConfig | SkipOrFail, NewSCBState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + - + +

OPTIONAL

+ +

This function is called before + + init_per_testcase if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + init_per_testcase function instead.

+ +

Note that it is not possible to add SCB's here right now, + that feature might be added later, + but it would right now break backwards compatability.

+ + + + + Module:post_end_per_testcase(TestcaseName, Config, Return, SCBState) + -> Result + Called after end_per_testcase + + TestcaseName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + SCBState = NewSCBState = term() + Result = {NewReturn, NewSCBState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + end_per_testcase if it exists. It behaves the same way as + + post_init_per_suite, but for the + + end_per_testcase function instead.

+ + + + + Module:pre_end_per_group(GroupName, Config, SCBState) -> + Result + Called before end_per_group + + GroupName = atom() + Config = NewConfig = [{Key,Value}] + SCBState = NewSCBState = term() + Result = {NewConfig | SkipOrFail, NewSCBState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + end_per_group if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + end_per_group function instead.

+ + + + + Module:post_end_per_group(GroupName, Config, Return, SCBState) -> + Result + Called after end_per_group + + GroupName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + SCBState = NewSCBState = term() + Result = {NewReturn, NewSCBState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + end_per_group if it exists. It behaves the same way as + + post_init_per_suite, but for the + + end_per_group function instead.

+ + + + + Module:pre_end_per_suite(SuiteName, Config, SCBState) -> + Result + Called before end_per_suite + + SuiteName = atom() + Config = NewConfig = [{Key,Value}] + SCBState = NewSCBState = term() + Result = {NewConfig | SkipOrFail, NewSCBState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + end_per_suite if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + end_per_suite function instead.

+ + + + + Module:post_end_per_suite(SuiteName, Config, Return, SCBState) -> + Result + Called after end_per_suite + + SuiteName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + SCBState = NewSCBState = term() + Result = {NewReturn, NewSCBState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + end_per_suite if it exists. It behaves the same way as + + post_init_per_suite, but for the + + end_per_suite function instead.

+ + + + + Module:on_tc_fail(TestcaseName, Reason, SCBState) -> + NewSCBState + Called after the SCB scope ends + + TestcaseName = init_per_suite | end_per_suite | + init_per_group | end_per_group | atom() + Reason = term() + SCBState = NewSCBState = term() + + + +

OPTIONAL

+ +

This function is called whenever a testcase fails. + It is called after the post function has been called for + the testcase which failed. i.e. + if init_per_suite fails this function is called after + + post_init_per_suite, and if a testcase fails it is called + after + post_end_per_testcase.

+ +

The data which comes with the Reason follows the same format as the + FailReason + in the tc_done event. + See Event Handling + in the User's Guide for details.

+ + + + + Module:on_tc_skip(TestcaseName, Reason, SCBState) -> + NewSCBState + Called after the SCB scope ends + + TestcaseName = end_per_suite | init_per_group | + end_per_group | atom() + Reason = {tc_auto_skip | tc_user_skip, term()} + SCBState = NewSCBState = term() + + + +

OPTIONAL

+ +

This function is called whenever a testcase is skipped. + It is called after the post function has been called for the + testcase which was skipped. + i.e. if init_per_group is skipped this function is called after + post_init_per_group + , and if a testcase is skipped it is called after + post_end_per_testcase + .

+ +

The data which comes with the Reason follows the same format as + tc_auto_skip + and + tc_user_skip events. + See Event Handling + in the User's Guide for details.

+ + + + + Module:terminate(SCBState) + Called after the SCB scope ends + + SCBState = term() + + + +

OPTIONAL

+ +

This function is called at the end of an SCB's + scope. +

+ + + + + Module:id(Opts) -> Id + Called before the init function of an SCB + + Opts = term() + Id = term() + + + +

OPTIONAL

+ +

The Id is used to uniquely identify an SCB instance, + if two SCB's return the same Id the second SCB is ignored + and subsequent calls to the SCB will only be made to the first + instance. For more information see + Installing an SCB + in the User's Guide. +

+ +

This function should NOT have any side effects as it might + be called multiple times by Common Test.

+ +

If not implemented the SCB will act as if this function returned a + call to make_ref/0.

+ - diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index 904876ac46..a01feb59d1 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -61,6 +61,7 @@ itself.

+ Usage

Event handlers may be installed by means of an event_handler start flag (ct_run) or option (ct:run_test/1), where the @@ -120,6 +121,7 @@ node the event has originated from (only relevant for CT Master event handlers). data is specific for the particular event.

+

General events:

@@ -172,6 +174,7 @@ are also given.

+ #event{name = tc_done, data = {Suite,FuncOrGroup,Result}}

Suite = atom(), name of the suite.

FuncOrGroup = Func | {Conf,GroupName,GroupProperties}

@@ -181,12 +184,14 @@ (unknown if init- or end function times out).

GroupProperties = list(), list of execution properties for the group.

Result = ok | {skipped,SkipReason} | {failed,FailReason}, the result.

+

SkipReason = {require_failed,RequireInfo} | {require_failed_in_suite0,RequireInfo} | {failed,{Suite,init_per_testcase,FailInfo}} | UserTerm, the reason why the case has been skipped.

-

FailReason = {error,FailInfo} | + +

FailReason = {error,FailInfo} | {error,{RunTimeError,StackTrace}} | {timetrap_timeout,integer()} | {failed,{Suite,end_per_testcase,FailInfo}}, reason for failure.

@@ -209,6 +214,7 @@ end_per_testcase for the case failed.

+ #event{name = tc_auto_skip, data = {Suite,Func,Reason}}

Suite = atom(), the name of the suite.

Func = atom(), the name of the test case or configuration function.

@@ -234,7 +240,8 @@ skipped because of init_per_testcase failing, since that information is carried with the tc_done event.

- + + #event{name = tc_user_skip, data = {Suite,TestCase,Comment}}

Suite = atom(), name of the suite.

TestCase = atom(), name of the test case.

diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 53a4cb1bbf..1a09ec1da2 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -75,6 +75,7 @@ + diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 94fcf6bf01..81e752680b 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -105,6 +105,7 @@ RPC from a remote node.

+
Running tests from the OS command line @@ -147,6 +148,8 @@ event handlers. ]]>, to install event handlers including start arguments. + ]]>, to install + Suite Callbacks including start arguments. , specifies include directories (see above). , disables the automatic test suite compilation feature (see above). ]]>, extends timetrap @@ -333,8 +336,8 @@ with dir.

+
- Using test specifications

The most flexible way to specify what to test, is to use a so @@ -440,6 +443,9 @@ {event_handler, NodeRefs, EventHandlers}. {event_handler, EventHandlers, InitArgs}. {event_handler, NodeRefs, EventHandlers, InitArgs}. + + {suite_callbacks, SCBModules}. + {suite_callbacks, NodeRefs, SCBModules}.

Test terms:

@@ -478,6 +484,9 @@
       LogDir        = string()
       EventHandlers = atom() | [atom()]
       InitArgs      = [term()]
+      SCBModules    = [SCBModule | {SCBModule, SCBInitArgs}]
+      SCBModule     = atom()
+      SCBInitArgs   = term()
       DirRef        = DirAlias | Dir
       Suites        = atom() | [atom()] | all
       Suite         = atom()
diff --git a/lib/common_test/doc/src/suite_callbacks_chapter.xml b/lib/common_test/doc/src/suite_callbacks_chapter.xml
new file mode 100644
index 0000000000..89f78898d4
--- /dev/null
+++ b/lib/common_test/doc/src/suite_callbacks_chapter.xml
@@ -0,0 +1,394 @@
+
+
+
+
+  

+    
+      20112011
+      Ericsson AB. All Rights Reserved.
+    
+    
+      The contents of this file are subject to the Erlang Public License,
+      Version 1.1, (the "License"); you may not use this file except in
+      compliance with the License. You should have received a copy of the
+      Erlang Public License along with this software. If not, it can be
+      retrieved online at http://www.erlang.org/.
+
+      Software distributed under the License is distributed on an "AS IS"
+      basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+      the License for the specific language governing rights and limitations
+      under the License.
+
+    
+
+    Suite Callbacks
+    Lukas Larsson
+    
+    
+    
+    suite_callbacks_chapter.xml
+  

+
+  
+  

+    General
+    
This feature is in alpha release right now. This means that the 
+	interface may change in the future and that there may be bugs. We 
+	encourage you to use this feature, but be prepared 
+	that there might be bugs and that the interface might change
+	inbetween releases.

+    

+      The Suite Callback (henceforth called SCB) framework allows 
+      extensions of the default behaviour of Common Test by means of callbacks 
+      before and after all test suite calls. SCBs allow advanced Common Test
+      users to abstract out behaviour which is common to multiple test suites
+      without littering all test suites with library calls. Some example 
+      usages are: logging, starting and monitoring external systems, 
+      building C files needed by the tests and much more!

+
+    
In brief, Suite Callbacks allows you to:

+
+    
+      Manipulate the runtime config before each suite 
+      configuration calls
+      Manipulate the return of all suite configuration calls and in 
+      extension the result of the test themselves.
+    
+    
+    
The following sections describe how to use SCBs, when they are run 
+      and how to manipulate your test results in an SCB

+
+    
When executing within an SCB all timetraps are shutoff. So
+	if your SCB never returns, the entire test run will be stalled!

+    
+
+  

+  
+  
+  

+    Installing an SCB
+    
There are multiple ways to install an SCB in your test run. You can do it
+      for all tests in a run, for specific test suites and for specific groups 
+      within a test suite. If you want an SCB to be present in all test suites 
+      within your testrun there are three different ways to accomplish that.

+
+    
+      Add -suite_callbacks as an argument to 
+      ct_run. 
+      To add multiple SCBs using this method append them to each other
+      using the keyword and, i.e. 
+      ct_run -suite_callbacks scb1 [{debug,true}] and scb2 ....
+      Add the suite_callbacks tag to your 
+      
+      Test Specification
+      Add the suite_callbacks tag to your call to 
+      ct:run_test/1
+    
+
+    
You can also add SCBs within a test suite. This is done by returning
+    {suite_callbacks,[SCB]} in the config list from 
+    suite/0,
+    
+      init_per_suite/1 or
+      
+    init_per_group/2. SCB in this case can be either
+    only the module name of the SCB or a tuple with the module name and the
+    initial arguments to the SCB. Eg:
+    {suite_callbacks,[my_scb_module]} or 
+    {suite_callbacks,[{my_scb_module,[{debug,true}]}]}

+
+    

+      Overriding SCBs
+      
By default each installation of an SCB will cause a new instance of it
+	to be activated. This can cause problems if you want to be able to 
+	override SCBs in testspecifications while still having them in the
+	suite info function. The 
+	id/1
+	callback exists to address this problem. By returning the same
+	id in both places, Common Test knows that this SCB
+	has already been installed and will not try to install it again.

+    

+
+  

+
+  
+  

+    SCB Scope
+    
Once the SCB is installed into a certain test run it will be there until
+      it's scope is expired. The scope of an SCB depends on when it is 
+      installed.
+      The init/2 is 
+      called at the beginning of the scope and the 
+      terminate/1
+     function is called when the scope ends.

+    
+      
+	SCB Installed in
+	SCB scope begins before
+	SCB scope ends after
+      
+      
+	ct_run
+	the first test suite is to be run.
+	the last test suite has been run.
+      
+      
+	ct:run_test
+	the first test suite is to be run.
+	the last test suite has been run.
+      
+      
+	
+	  Test Specification
+	the first test suite is to be run.
+	the last test suite has been run.
+      
+      
+	suite/0
+	
+	
+	    pre_init_per_suite/3 is called.
+	
+	  post_end_per_suite/4 has been called for that test suite.
+      
+      
+	
+	  init_per_suite/1
+	
+	    post_init_per_suite/4 is called.
+	
+	  post_end_per_suite/4 has been called for that test suite.
+      
+      
+	
+	  init_per_group/2
+	
+	    post_init_per_group/4 is called.
+	
+	  post_end_per_group/4 has been called for that group.
+      
+    Scope of an SCB
+    

+    
+    

+      CTH Processes and Tables
+      
CTHs are run with the same process scoping as normal test suites
+	i.e. a different process will execute the init_per_suite hooks then the
+	init_per_group or per_testcase hooks. So if you want to spawn a 
+	process in the CTH you cannot link with the CTH process as it will exit 
+	after the post hook ends. Also if you for some reason need an ETS 
+	table with your CTH, you will have to spawn a process which handles 
+	it.

+    

+    
+  

+
+  
+  

+    Manipulating tests
+    
It is through SCB's possible to manipulate the results of tests and 
+    configuration functions. The main purpose of doing this with SCBs is to
+    allow common patterns to be abstracted out from test test suites and applied to
+    multiple test suites without duplicating any code. All of the callback
+    functions for an SCB follow a common interface, this interface is 
+    described below.

+
+    
+    

+      Pre test manipulation
+      

+	It is possible in an SCB to hook in behaviour before 
+	init_per_suite, 
+	init_per_group, 
+	init_per_testcase, 
+	end_per_group and 
+	end_per_suite. 
+	This is done in the SCB functions called pre_<name of function>.
+	All of these function take the same three arguments: Name, 
+	Config and SCBState. The return value of the SCB function
+	is always a combination of an result for the suite/group/test and an 
+	updated SCBState. If you want the test suite to continue on 
+	executing you should return the config list which you want the test to 
+	use as the result. If you for some reason want to skip/fail the test, 
+	return a tuple with skip or fail and a reason as the 
+	result. Example:
+      

+      pre_init_per_suite(SuiteName, Config, SCBState) ->
+  case db:connect() of
+    {error,_Reason} ->
+      {{fail, "Could not connect to DB"}, SCBState};
+    {ok, Handle} ->
+      {[{db_handle, Handle} | Config], SCBState#state{ handle = Handle }}
+  end.
+	
+    

+    
+    
+    

+      Post test manipulation
+      
It is also possible in an SCB to hook in behaviour after 
+      init_per_suite, 
+      init_per_group, 
+      end_per_testcase, 
+      end_per_group and 
+      end_per_suite.
+      This is done in the SCB functions called post_<name of function>. 
+      All of these function take the same four arguments: Name, 
+      Config, Return and SCBState. Config in this
+      case is the same Config as the testcase is called with. 
+      Return is the value returned by the testcase. If the testcase 
+      failed by crashing, Return will be 
+      {'EXIT',{{Error,Reason},Stacktrace}}.

+      
+      
The return value of the SCB function is always a combination of an
+	result for the suite/group/test and an updated SCBState. If
+	you want the callback to not affect the outcome of the test you should
+	return the Return data as it is given to the SCB. You can also
+	modify the result of the test. By returning the Config list
+	with the tc_status element removed you can recover from a test 
+	failure. As in all the pre hooks, it is also possible to fail/skip
+	the test case in the post hook. Example: 

+
+      post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, SCBState) ->
+  case db:check_consistency() of
+    true ->
+      %% DB is good, pass the test.
+      {proplists:delete(tc_status, Config), SCBState};
+    false ->
+      %% DB is not good, mark as skipped instead of failing
+      {{skip, "DB is inconsisten!"}, SCBState}
+  end;
+post_end_per_testcase(_TC, Config, Return, SCBState) ->
+  %% Do nothing if tc does not crash.
+  {Return, SCBState}.
+
+      Recovering from a testcase failure using SCBs should only be done as
+	a last resort. If used wrongly it could become very difficult to 
+	determine which tests pass or fail in a test run
+  
+    

+
+    
+    

+      Skip and Fail
+      

+	After any post hook has been executed for all installed SCBs, 
+	on_tc_fail
+	or on_tc_skip 
+	might be called if the testcase failed or was skipped 
+	respectively. You cannot affect the outcome of the tests any further at 
+	this point. 
+      

+    

+
+  

+
+  
+  

+     Example SCB
+     
The SCB below will log information about a test run into a format 
+       parseable by file:consult/1.
+     

+     %%% @doc Common Test Example Suite Callback module.
+-module(example_scb).
+
+%% Suite Callbacks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-record(state, { file_handle, total, suite_total, ts, tcs, data }).
+
+%% @doc Return a unique id for this SCB.
+id(Opts) ->
+  proplists:get_value(filename, Opts, "/tmp/file.log").
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. 
+init(Id, Opts) ->
+    {ok,D} = file:open(Id,[write]),
+    #state{ file_handle = D, total = 0, data = [] }.
+
+%% @doc Called before init_per_suite is called. 
+pre_init_per_suite(Suite,Config,State) ->
+    {Config, State#state{ suite_total = 0, tcs = [] }}.
+
+%% @doc Called after init_per_suite.
+post_init_per_suite(Suite,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before end_per_suite. 
+pre_end_per_suite(Suite,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after end_per_suite. 
+post_end_per_suite(Suite,Config,Return,State) ->
+    Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)},
+    {Return, State#state{ data = [Data | State#state.data] ,
+                          total = State#state.total + State#state.suite_total } }.
+
+%% @doc Called before each init_per_group.
+pre_init_per_group(Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each init_per_group.
+post_init_per_group(Group,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called after each end_per_group. 
+pre_end_per_group(Group,Config,State) ->
+    {Config, State}.
+
+%% @doc Called after each end_per_group. 
+post_end_per_group(Group,Config,Return,State) ->
+    {Return, State}.
+
+%% @doc Called before each test case.
+pre_init_per_testcase(TC,Config,State) ->
+    {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.
+
+%% @doc Called after each test case.
+post_end_per_testcase(TC,Config,Return,State) ->
+    TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)},
+    {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
+on_tc_fail(TC, Reason, State) ->
+    State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing.  
+on_tc_skip(TC, Reason, State) ->
+    State.
+
+%% @doc Called when the scope of the SCB is done
+terminate(State) ->
+    io:format(State#state.file_handle, "~p.~n",
+               [{test_run, State#state.total, State#state.data}]),
+    file:close(State#state.file_handle),
+    ok.
+  

+
+
+
+
+
+
diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml
index 5afec6de6a..76493d3616 100644
--- a/lib/common_test/doc/src/write_test_chapter.xml
+++ b/lib/common_test/doc/src/write_test_chapter.xml
@@ -115,6 +115,7 @@
+
Init and end per test case -- cgit v1.2.3 From 044f768b6545234461f173e8a959379630723e8f Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Wed, 9 Feb 2011 16:39:29 +0100 Subject: Rename Suite Callback to Common Test hook in documentation --- lib/common_test/doc/src/Makefile | 4 +- lib/common_test/doc/src/common_test_app.xml | 12 +- lib/common_test/doc/src/ct_hooks.xml | 556 +++++++++++++++++++++ lib/common_test/doc/src/ct_hooks_chapter.xml | 401 +++++++++++++++ lib/common_test/doc/src/ct_suite_callbacks.xml | 551 -------------------- lib/common_test/doc/src/part.xml | 2 +- lib/common_test/doc/src/ref_man.xml | 2 +- lib/common_test/doc/src/run_test_chapter.xml | 14 +- .../doc/src/suite_callbacks_chapter.xml | 394 --------------- 9 files changed, 974 insertions(+), 962 deletions(-) create mode 100644 lib/common_test/doc/src/ct_hooks.xml create mode 100644 lib/common_test/doc/src/ct_hooks_chapter.xml delete mode 100644 lib/common_test/doc/src/ct_suite_callbacks.xml delete mode 100644 lib/common_test/doc/src/suite_callbacks_chapter.xml (limited to 'lib/common_test/doc') diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index a3f740852c..a914dd0c19 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -52,7 +52,7 @@ CT_XML_FILES = $(CT_MODULES:=.xml) XML_APPLICATION_FILES = ref_man.xml XML_REF1_FILES = ct_run.xml -XML_REF3_FILES = $(CT_XML_FILES) ct_suite_callbacks.xml +XML_REF3_FILES = $(CT_XML_FILES) ct_hooks.xml XML_REF6_FILES = common_test_app.xml XML_PART_FILES = part.xml @@ -71,7 +71,7 @@ XML_CHAPTER_FILES = \ cover_chapter.xml \ ct_master_chapter.xml \ event_handler_chapter.xml \ - suite_callbacks_chapter.xml \ + ct_hooks_chapter.xml \ dependencies_chapter.xml \ notes.xml \ notes_history.xml diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index a735dc85a8..b4e4b45d62 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -132,7 +132,7 @@ Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns} | {stylesheet,CSSFile} | - {suite_callbacks, SCBs} + {ct_hooks, CTHs} Time = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()} MilliSec = integer() @@ -144,9 +144,9 @@ UserData = term() Conns = [atom()] CSSFile = string() - SCBs = [SCBModule | {SCBModule, SCBInitArgs}] - SCBModule = atom() - SCBInitArgs = term() + CTHs = [CTHModule | {CTHModule, CTHInitArgs}] + CTHModule = atom() + CTHInitArgs = term() @@ -175,8 +175,8 @@ specify arbitrary test suite related information which can be read by calling ct:userdata/2.

-

The suite_callbacks tag specifies which - Suite Callbacks +

The ct_hooks tag specifies which + Common Test Hooks are to be run together with this suite.

Other tuples than the ones defined will simply be ignored.

diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml new file mode 100644 index 0000000000..0d59ce3b22 --- /dev/null +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -0,0 +1,556 @@ + + + + + +
+ + 20102010 + Ericsson AB. All Rights Reserved. + + + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + + + Common Test Hooks + Lukas Larsson + Lukas Larsson + + + + 2010-12-02 + PA1 + ct_hooks.sgml +
+ ct_hooks + A callback interface on top of Common Test + + + +

This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.

 + +

The Common Test Hook (henceforth called CTH) framework allows + extensions of the default behaviour of Common Test by means of callbacks + before and after all test suite calls. It is meant for advanced users of + Common Test which want to abstract out behaviour which is common to + multiple test suites.

+ +

In brief, Common Test Hooks allows you to:

+ + + Manipulate the runtime config before each suite + configuration call + Manipulate the return of all suite configuration calls and in + extension the result of the test themselves. + + +

The following sections describe the mandatory and optional CTH + functions Common Test will call during test execution. For more details + see Common Test Hooks in + the User's Guide.

+ +

For information about how to add a CTH to your suite see + Installing a CTH + in the User's Guide.

+ +

See the + Example CTH + in the User's Guide for a minimal example of a CTH.

+ + + +
+ CALLBACK FUNCTIONS +

The following functions define the callback interface + for a Common Test Hook.

+
+ + + + Module:init(Id, Opts) -> State + Initiates the Common Test Hook + + Id = reference() | term() + Opts = term() + State = term() + + + +

MANDATORY

+ +

Always called before any other callback function. + Use this to initiate any common state. + It should return a state for this CTH.

+ +

Id is the return value of + id/1, or a reference + (created using + make_ref/0) + if id/1 is not implemented. +

+ +

For details about when init is called see + scope + in the User's Guide.

+ + + + + + Module:pre_init_per_suite(SuiteName, Config, CTHState) -> + Result + Called before init_per_suite + + SuiteName = atom() + Config = NewConfig = [{Key,Value}] + CTHState = NewCTHState = term() + Result = {Return, NewCTHState} + Return = NewConfig | SkipOrFail + SkipOrFail = {fail, Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + init_per_suite if it exists. + It typically contains initialization/logging which needs to be done + before init_per_suite is called. + If {skip,Reason} or {fail,Reason} is returned, + init_per_suite and all test cases of the suite will be skipped and + Reason printed in the overview log of the suite.

+ +

SuiteName is the name of the suite to be run.

+ +

Config is the original config list of the test suite.

+ +

CTHState is the current internal state of the CTH.

+ +

Return is the result of the init_per_suite function. + If it is {skip,Reason} or {fail,Reason} + init_per_suite + will never be called, instead the initiation is considered + to be skipped/failed respectively. If a NewConfig list + is returned, + init_per_suite will be called with that NewConfig list. + See + Pre Hooks in the User's Guide for more details.

+ + +

Note that this function is only called if the CTH has been added + before init_per_suite is run, see + CTH Scoping + in the User's Guide for details.

+ + + + + Module:post_init_per_suite(SuiteName, Config, Return, CTHState) -> + Result + Called after init_per_suite + + SuiteName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail, Reason} | {skip, Reason} | term() + CTHState = NewCTHState = term() + Result = {NewReturn, NewCTHState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + init_per_suite if it exists. It typically contains extra + checks to make sure that all the correct dependencies have + been started correctly.

+ +

Return is what + init_per_suite + returned, i.e. {fail,Reason}, {skip,Reason}, a Config + list or a term describing how + init_per_suite + failed.

+ +

NewReturn is the possibly modified return value of + init_per_suite + . It is here possible to recover from a failure in + init_per_suite + by returning the ConfigList with the tc_status + element removed. See + Post Hooks in the User's Guide for more details.

+ +

CTHState is the current internal state of the CTH.

+ +

Note that this function is only called if the CTH has been added + before or in init_per_suite, see + CTH Scoping + in the User's Guide for details.

+ + + + + Module:pre_init_per_group(GroupName, Config, CTHState) -> + Result + Called before init_per_group + + GroupName = atom() + Config = NewConfig = [{Key,Value}] + CTHState = NewCTHState = term() + Result = {NewConfig | SkipOrFail, NewCTHState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + init_per_group if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + init_per_group instead.

+ + + + + Module:post_init_per_group(GroupName, Config, Return, CTHState) -> + Result + Called after init_per_group + + GroupName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + CTHState = NewCTHState = term() + Result = {NewReturn, NewCTHState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + init_per_group if it exists. It behaves the same way as + + post_init_per_suite, but for the + + init_per_group instead.

+ + + + + Module:pre_init_per_testcase(TestcaseName, Config, CTHState) -> + Result + Called before init_per_testcase + + TestcaseName = atom() + Config = NewConfig = [{Key,Value}] + CTHState = NewCTHState = term() + Result = {NewConfig | SkipOrFail, NewCTHState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + init_per_testcase if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + init_per_testcase function instead.

+ +

Note that it is not possible to add CTH's here right now, + that feature might be added later, + but it would right now break backwards compatability.

+ + + + + Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) + -> Result + Called after end_per_testcase + + TestcaseName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + CTHState = NewCTHState = term() + Result = {NewReturn, NewCTHState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + end_per_testcase if it exists. It behaves the same way as + + post_init_per_suite, but for the + + end_per_testcase function instead.

+ + + + + Module:pre_end_per_group(GroupName, Config, CTHState) -> + Result + Called before end_per_group + + GroupName = atom() + Config = NewConfig = [{Key,Value}] + CTHState = NewCTHState = term() + Result = {NewConfig | SkipOrFail, NewCTHState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + end_per_group if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + end_per_group function instead.

+ + + + + Module:post_end_per_group(GroupName, Config, Return, CTHState) -> + Result + Called after end_per_group + + GroupName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + CTHState = NewCTHState = term() + Result = {NewReturn, NewCTHState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + end_per_group if it exists. It behaves the same way as + + post_init_per_suite, but for the + + end_per_group function instead.

+ + + + + Module:pre_end_per_suite(SuiteName, Config, CTHState) -> + Result + Called before end_per_suite + + SuiteName = atom() + Config = NewConfig = [{Key,Value}] + CTHState = NewCTHState = term() + Result = {NewConfig | SkipOrFail, NewCTHState} + SkipOrFail = {fail,Reason} | {skip, Reason} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called before + + end_per_suite if it exists. It behaves the same way as + + pre_init_per_suite, but for the + + end_per_suite function instead.

+ + + + + Module:post_end_per_suite(SuiteName, Config, Return, CTHState) -> + Result + Called after end_per_suite + + SuiteName = atom() + Config = [{Key,Value}] + Return = NewReturn = Config | SkipOrFail | term() + SkipOrFail = {fail,Reason} | {skip, Reason} + CTHState = NewCTHState = term() + Result = {NewReturn, NewCTHState} + Key = atom() + Value = term() + Reason = term() + + + +

OPTIONAL

+ +

This function is called after + + end_per_suite if it exists. It behaves the same way as + + post_init_per_suite, but for the + + end_per_suite function instead.

+ + + + + Module:on_tc_fail(TestcaseName, Reason, CTHState) -> + NewCTHState + Called after the CTH scope ends + + TestcaseName = init_per_suite | end_per_suite | + init_per_group | end_per_group | atom() + Reason = term() + CTHState = NewCTHState = term() + + + +

OPTIONAL

+ +

This function is called whenever a testcase fails. + It is called after the post function has been called for + the testcase which failed. i.e. + if init_per_suite fails this function is called after + + post_init_per_suite, and if a testcase fails it is called + after + post_end_per_testcase.

+ +

The data which comes with the Reason follows the same format as the + FailReason + in the tc_done event. + See Event Handling + in the User's Guide for details.

+ + + + + Module:on_tc_skip(TestcaseName, Reason, CTHState) -> + NewCTHState + Called after the CTH scope ends + + TestcaseName = end_per_suite | init_per_group | + end_per_group | atom() + Reason = {tc_auto_skip | tc_user_skip, term()} + CTHState = NewCTHState = term() + + + +

OPTIONAL

+ +

This function is called whenever a testcase is skipped. + It is called after the post function has been called for the + testcase which was skipped. + i.e. if init_per_group is skipped this function is called after + post_init_per_group + , and if a testcase is skipped it is called after + post_end_per_testcase + .

+ +

The data which comes with the Reason follows the same format as + tc_auto_skip + and + tc_user_skip events. + See Event Handling + in the User's Guide for details.

+ + + + + Module:terminate(CTHState) + Called after the CTH scope ends + + CTHState = term() + + + +

OPTIONAL

+ +

This function is called at the end of a CTH's + scope. +

+ + + + + Module:id(Opts) -> Id + Called before the init function of a CTH + + Opts = term() + Id = term() + + + +

OPTIONAL

+ +

The Id is used to uniquely identify a CTH instance, + if two CTH's return the same Id the second CTH is ignored + and subsequent calls to the CTH will only be made to the first + instance. For more information see + Installing a CTH + in the User's Guide. +

+ +

This function should NOT have any side effects as it might + be called multiple times by Common Test.

+ +

If not implemented the CTH will act as if this function returned a + call to make_ref/0.

+ + + + + + + + diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml new file mode 100644 index 0000000000..fc5ab48e1b --- /dev/null +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -0,0 +1,401 @@ + + + + +
+ + 20112011 + Ericsson AB. All Rights Reserved. + + + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + + + Common Test Hooks + Lukas Larsson + + + + ct_hooks_chapter.xml +
+ + +
+ General +

This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.

 +

+ The Common Test Hook (henceforth called CTH) framework allows + extensions of the default behaviour of Common Test by means of hooks + before and after all test suite calls. CTHs allow advanced Common Test + users to abstract out behaviour which is common to multiple test suites + without littering all test suites with library calls. Some example + usages are: logging, starting and monitoring external systems, + building C files needed by the tests and much more!

+ +

In brief, Common Test Hooks allows you to:

+ + + Manipulate the runtime config before each suite + configuration call + Manipulate the return of all suite configuration calls and in + extension the result of the test themselves. + + +

The following sections describe how to use CTHs, when they are run + and how to manipulate your test results in a CTH

+ +

When executing within a CTH all timetraps are shutoff. So + if your CTH never returns, the entire test run will be stalled!

+ + +
+ + +
+ Installing a CTH +

There are multiple ways to install a CTH in your test run. You can do it + for all tests in a run, for specific test suites and for specific groups + within a test suite. If you want a CTH to be present in all test suites + within your test run there are three different ways to accomplish that. +

+ + + Add -ct_hooks as an argument to + ct_run. + To add multiple CTHs using this method append them to each other + using the keyword and, i.e. + ct_run -ct_hooks cth1 [{debug,true}] and cth2 .... + Add the ct_hooks tag to your + + Test Specification + Add the ct_hooks tag to your call to + ct:run_test/1 + + +

You can also add CTHs within a test suite. This is done by returning + {ct_hooks,[CTH]} in the config list from + suite/0, + + init_per_suite/1 or + + init_per_group/2. CTH in this case can be either + only the module name of the CTH or a tuple with the module name and the + initial arguments to the CTH. Eg: + {ct_hooks,[my_cth_module]} or + {ct_hooks,[{my_cth_module,[{debug,true}]}]}

+ +
+ Overriding CTHs +

By default each installation of a CTH will cause a new instance of it + to be activated. This can cause problems if you want to be able to + override CTHs in test specifications while still having them in the + suite info function. The + id/1 + callback exists to address this problem. By returning the same + id in both places, Common Test knows that this CTH + has already been installed and will not try to install it again.

+
+ +
+ + +
+ CTH Scope +

Once the CTH is installed into a certain test run it will be there until + its scope is expired. The scope of a CTH depends on when it is + installed. + The init/2 is + called at the beginning of the scope and the + terminate/1 + function is called when the scope ends.

+ + + CTH Installed in + CTH scope begins before + CTH scope ends after + + + ct_run + the first test suite is to be run. + the last test suite has been run. + + + ct:run_test + the first test suite is to be run. + the last test suite has been run. + + + + Test Specification + the first test suite is to be run. + the last test suite has been run. + + + suite/0 + + + pre_init_per_suite/3 is called. + + post_end_per_suite/4 has been called for that test suite. + + + + init_per_suite/1 + + post_init_per_suite/4 is called. + + post_end_per_suite/4 has been called for that test suite. + + + + init_per_group/2 + + post_init_per_group/4 is called. + + post_end_per_group/4 has been called for that group. + + Scope of a CTH +
+ +
+ CTH Processes and Tables +

CTHs are run with the same process scoping as normal test suites + i.e. a different process will execute the init_per_suite hooks then the + init_per_group or per_testcase hooks. So if you want to spawn a + process in the CTH you cannot link with the CTH process as it will exit + after the post hook ends. Also if you for some reason need an ETS + table with your CTH, you will have to spawn a process which handles + it.

+
+ +
+ + +
+ Manipulating tests +

It is through CTHs possible to manipulate the results of tests and + configuration functions. The main purpose of doing this with CTHs is to + allow common patterns to be abstracted out from test test suites and applied to + multiple test suites without duplicating any code. All of the callback + functions for a CTH follow a common interface, this interface is + described below.

+ +

It is only possible to hook into test function which exists in the test + suite. So in order for a CTH to hook in before + init_per_suite, + the init_per_suite + function must exist in the test suite.

+ + +
+ Pre Hooks +

+ It is possible in a CTH to hook in behaviour before + init_per_suite, + init_per_group, + init_per_testcase, + end_per_group and + end_per_suite. + This is done in the CTH functions called pre_<name of function>. + All of these functions take the same three arguments: Name, + Config and CTHState. The return value of the CTH function + is always a combination of an result for the suite/group/test and an + updated CTHState. If you want the test suite to continue on + executing you should return the config list which you want the test to + use as the result. If you for some reason want to skip/fail the test, + return a tuple with skip or fail and a reason as the + result. Example: +

+ pre_init_per_suite(SuiteName, Config, CTHState) -> + case db:connect() of + {error,_Reason} -> + {{fail, "Could not connect to DB"}, CTHState}; + {ok, Handle} -> + {[{db_handle, Handle} | Config], CTHState#state{ handle = Handle }} + end. + +
+ + +
+ Post Hooks +

It is also possible in a CTH to hook in behaviour after + init_per_suite, + init_per_group, + end_per_testcase, + end_per_group and + end_per_suite. + This is done in the CTH functions called post_<name of function>. + All of these function take the same four arguments: Name, + Config, Return and CTHState. Config in this + case is the same Config as the testcase is called with. + Return is the value returned by the testcase. If the testcase + failed by crashing, Return will be + {'EXIT',{{Error,Reason},Stacktrace}}.

+ +

The return value of the CTH function is always a combination of an + result for the suite/group/test and an updated CTHState. If + you want the callback to not affect the outcome of the test you should + return the Return data as it is given to the CTH. You can also + modify the result of the test. By returning the Config list + with the tc_status element removed you can recover from a test + failure. As in all the pre hooks, it is also possible to fail/skip + the test case in the post hook. Example:

+ + post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + case db:check_consistency() of + true -> + %% DB is good, pass the test. + {proplists:delete(tc_status, Config), CTHState}; + false -> + %% DB is not good, mark as skipped instead of failing + {{skip, "DB is inconsisten!"}, CTHState} + end; +post_end_per_testcase(_TC, Config, Return, CTHState) -> + %% Do nothing if tc does not crash. + {Return, CTHState}. + + Recovering from a testcase failure using CTHs should only be done as + a last resort. If used wrongly it could become very difficult to + determine which tests pass or fail in a test run + +
+ + +
+ Skip and Fail hooks +

+ After any post hook has been executed for all installed CTHs, + on_tc_fail + or on_tc_skip + might be called if the testcase failed or was skipped + respectively. You cannot affect the outcome of the tests any further at + this point. +

+
+ +
+ + +
+ Example CTH +

The CTH below will log information about a test run into a format + parseable by file:consult/1. +

+ %%% @doc Common Test Example Common Test Hook module. +-module(example_cth). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3]). + +-export([terminate/1]). + +-record(state, { file_handle, total, suite_total, ts, tcs, data }). + +%% @doc Return a unique id for this CTH. +id(Opts) -> + proplists:get_value(filename, Opts, "/tmp/file.log"). + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, Opts) -> + {ok,D} = file:open(Id,[write]), + #state{ file_handle = D, total = 0, data = [] }. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{ suite_total = 0, tcs = [] }}. + +%% @doc Called after init_per_suite. +post_init_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(Suite,Config,Return,State) -> + Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)}, + {Return, State#state{ data = [Data | State#state.data] , + total = State#state.total + State#state.suite_total } }. + +%% @doc Called before each init_per_group. +pre_init_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called before each test case. +pre_init_per_testcase(TC,Config,State) -> + {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. + +%% @doc Called after each test case. +post_end_per_testcase(TC,Config,Return,State) -> + TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(TC, Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. +on_tc_skip(TC, Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(State) -> + io:format(State#state.file_handle, "~p.~n", + [{test_run, State#state.total, State#state.data}]), + file:close(State#state.file_handle), + ok. +
+ + + + + + diff --git a/lib/common_test/doc/src/ct_suite_callbacks.xml b/lib/common_test/doc/src/ct_suite_callbacks.xml deleted file mode 100644 index cea5804825..0000000000 --- a/lib/common_test/doc/src/ct_suite_callbacks.xml +++ /dev/null @@ -1,551 +0,0 @@ - - - - - -
- - 20102010 - Ericsson AB. All Rights Reserved. - - - The contents of this file are subject to the Erlang Public License, - Version 1.1, (the "License"); you may not use this file except in - compliance with the License. You should have received a copy of the - Erlang Public License along with this software. If not, it can be - retrieved online at http://www.erlang.org/. - - Software distributed under the License is distributed on an "AS IS" - basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - the License for the specific language governing rights and limitations - under the License. - - - - Suite Callbacks - Lukas Larsson - Lukas Larsson - - - - 2010-12-02 - PA1 - suite_callback.sgml -
- ct_suite_callback - A callback interface on top of Common Test - - - -

This feature is in alpha release right now. This means that the - interface may change in the future and that there may be bugs. We - encourage you to use this feature, but be prepared - that there might be bugs and that the interface might change - inbetween releases.

 - -

The Suite Callback (henceforth called SCB) framework allows - extensions of the default behaviour of Common Test by means of callbacks - before and after all test suite calls. It is meant for advanced users of - Common Test which want to abstract out behaviour which is common to - multiple test suites.

- -

In brief, Suite Callbacks allows you to:

- - - Manipulate the runtime config before each suite - configuration calls - Manipulate the return of all suite configuration calls and in - extension the result of the test themselves. - - -

The following sections describe the mandatory and optional SCB - functions Common Test will call during test execution. For more details - see Suite Callbacks in - the User's Guide.

- -

For information about how to add a SCB to your suite see - Installing an SCB - in the User's Guide.

- -

See the - Example SCB - in the User's Guide for a minimal example of an SCB.

- - - -
- CALLBACK FUNCTIONS -

The following functions define the callback interface - for a suite callback.

-
- - - - Module:init(Id, Opts) -> State - Initiates the Suite Callback - - Id = reference() | term() - Opts = term() - State = term() - - - -

MANDATORY

- -

Always called before any other callback function. - Use this to initiate any common state. - It should return a state for this SCB.

- -

Id is the return value of - id/1, or a reference - if id/1 is not implemented. -

- -

For details about when init is called see - scope - in the User's Guide.

- - - - - - Module:pre_init_per_suite(SuiteName, Config, SCBState) -> - Result - Called before init_per_suite - - SuiteName = atom() - Config = NewConfig = [{Key,Value}] - SCBState = NewSCBState = term() - Result = {Return, NewSCBState} - Return = NewConfig | SkipOrFail - SkipOrFail = {fail, Reason} | {skip, Reason} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called before - - init_per_suite if it exists. - It typically contains initialization/logging which needs to be done - before init_per_suite is called. - If {skip,Reason} or {fail,Reason} is returned, - init_per_suite and all test cases of the suite will be skipped and - Reason printed in the overview log of the suite.

- -

SuiteName is the name of the suite to be run.

- -

Config is the original config list of the test suite.

- -

SCBState is the current internal state of the SCB.

- -

Return is the result of the init_per_suite function. - If it is {skip,Reason} or {fail,Reason} - init_per_suite - will never be called, instead the initiation is considered - to be skipped/failed respectively. If a NewConfig list - is returned, - init_per_suite will be called with that NewConfig list. - See - Manipulating tests in the User's Guide for more details.

- - -

Note that this function is only called if the SCB has been added - before init_per_suite is run, see - SCB Scoping - in the User's Guide for details.

- - - - - Module:post_init_per_suite(SuiteName, Config, Return, SCBState) -> - Result - Called after init_per_suite - - SuiteName = atom() - Config = [{Key,Value}] - Return = NewReturn = Config | SkipOrFail | term() - SkipOrFail = {fail, Reason} | {skip, Reason} | term() - SCBState = NewSCBState = term() - Result = {NewReturn, NewSCBState} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

Return is what - init_per_suite - returned, i.e. {fail,Reason}, {skip,Reason}, a Config - list or a term describing how - init_per_suite - failed.

- -

NewReturn is the possibly modified return value of - init_per_suite - . It is here possible to recover from a failure in - init_per_suite - by returning the ConfigList with the tc_status - element removed.

- -

SCBState is the current internal state of the SCB.

- -

This function is called after - - init_per_suite if it exists.

- -

Note that this function is only called if the SCB has been added - before or in init_per_suite, see - SCB Scoping - in the User's Guide for details.

- - - - - Module:pre_init_per_group(GroupName, Config, SCBState) -> - Result - Called before init_per_group - - GroupName = atom() - Config = NewConfig = [{Key,Value}] - SCBState = NewSCBState = term() - Result = {NewConfig | SkipOrFail, NewSCBState} - SkipOrFail = {fail,Reason} | {skip, Reason} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called before - - init_per_group if it exists. It behaves the same way as - - pre_init_per_suite, but for the - - init_per_group instead.

- - - - - Module:post_init_per_group(GroupName, Config, Return, SCBState) -> - Result - Called after init_per_group - - GroupName = atom() - Config = [{Key,Value}] - Return = NewReturn = Config | SkipOrFail | term() - SkipOrFail = {fail,Reason} | {skip, Reason} - SCBState = NewSCBState = term() - Result = {NewReturn, NewSCBState} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called after - - init_per_group if it exists. It behaves the same way as - - post_init_per_suite, but for the - - init_per_group instead.

- - - - - Module:pre_init_per_testcase(TestcaseName, Config, SCBState) -> - Result - Called before init_per_testcase - - TestcaseName = atom() - Config = NewConfig = [{Key,Value}] - SCBState = NewSCBState = term() - Result = {NewConfig | SkipOrFail, NewSCBState} - SkipOrFail = {fail,Reason} | {skip, Reason} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called before - - init_per_testcase if it exists. It behaves the same way as - - pre_init_per_suite, but for the - - init_per_testcase function instead.

- -

Note that it is not possible to add SCB's here right now, - that feature might be added later, - but it would right now break backwards compatability.

- - - - - Module:post_end_per_testcase(TestcaseName, Config, Return, SCBState) - -> Result - Called after end_per_testcase - - TestcaseName = atom() - Config = [{Key,Value}] - Return = NewReturn = Config | SkipOrFail | term() - SkipOrFail = {fail,Reason} | {skip, Reason} - SCBState = NewSCBState = term() - Result = {NewReturn, NewSCBState} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called after - - end_per_testcase if it exists. It behaves the same way as - - post_init_per_suite, but for the - - end_per_testcase function instead.

- - - - - Module:pre_end_per_group(GroupName, Config, SCBState) -> - Result - Called before end_per_group - - GroupName = atom() - Config = NewConfig = [{Key,Value}] - SCBState = NewSCBState = term() - Result = {NewConfig | SkipOrFail, NewSCBState} - SkipOrFail = {fail,Reason} | {skip, Reason} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called before - - end_per_group if it exists. It behaves the same way as - - pre_init_per_suite, but for the - - end_per_group function instead.

- - - - - Module:post_end_per_group(GroupName, Config, Return, SCBState) -> - Result - Called after end_per_group - - GroupName = atom() - Config = [{Key,Value}] - Return = NewReturn = Config | SkipOrFail | term() - SkipOrFail = {fail,Reason} | {skip, Reason} - SCBState = NewSCBState = term() - Result = {NewReturn, NewSCBState} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called after - - end_per_group if it exists. It behaves the same way as - - post_init_per_suite, but for the - - end_per_group function instead.

- - - - - Module:pre_end_per_suite(SuiteName, Config, SCBState) -> - Result - Called before end_per_suite - - SuiteName = atom() - Config = NewConfig = [{Key,Value}] - SCBState = NewSCBState = term() - Result = {NewConfig | SkipOrFail, NewSCBState} - SkipOrFail = {fail,Reason} | {skip, Reason} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called before - - end_per_suite if it exists. It behaves the same way as - - pre_init_per_suite, but for the - - end_per_suite function instead.

- - - - - Module:post_end_per_suite(SuiteName, Config, Return, SCBState) -> - Result - Called after end_per_suite - - SuiteName = atom() - Config = [{Key,Value}] - Return = NewReturn = Config | SkipOrFail | term() - SkipOrFail = {fail,Reason} | {skip, Reason} - SCBState = NewSCBState = term() - Result = {NewReturn, NewSCBState} - Key = atom() - Value = term() - Reason = term() - - - -

OPTIONAL

- -

This function is called after - - end_per_suite if it exists. It behaves the same way as - - post_init_per_suite, but for the - - end_per_suite function instead.

- - - - - Module:on_tc_fail(TestcaseName, Reason, SCBState) -> - NewSCBState - Called after the SCB scope ends - - TestcaseName = init_per_suite | end_per_suite | - init_per_group | end_per_group | atom() - Reason = term() - SCBState = NewSCBState = term() - - - -

OPTIONAL

- -

This function is called whenever a testcase fails. - It is called after the post function has been called for - the testcase which failed. i.e. - if init_per_suite fails this function is called after - - post_init_per_suite, and if a testcase fails it is called - after - post_end_per_testcase.

- -

The data which comes with the Reason follows the same format as the - FailReason - in the tc_done event. - See Event Handling - in the User's Guide for details.

- - - - - Module:on_tc_skip(TestcaseName, Reason, SCBState) -> - NewSCBState - Called after the SCB scope ends - - TestcaseName = end_per_suite | init_per_group | - end_per_group | atom() - Reason = {tc_auto_skip | tc_user_skip, term()} - SCBState = NewSCBState = term() - - - -

OPTIONAL

- -

This function is called whenever a testcase is skipped. - It is called after the post function has been called for the - testcase which was skipped. - i.e. if init_per_group is skipped this function is called after - post_init_per_group - , and if a testcase is skipped it is called after - post_end_per_testcase - .

- -

The data which comes with the Reason follows the same format as - tc_auto_skip - and - tc_user_skip events. - See Event Handling - in the User's Guide for details.

- - - - - Module:terminate(SCBState) - Called after the SCB scope ends - - SCBState = term() - - - -

OPTIONAL

- -

This function is called at the end of an SCB's - scope. -

- - - - - Module:id(Opts) -> Id - Called before the init function of an SCB - - Opts = term() - Id = term() - - - -

OPTIONAL

- -

The Id is used to uniquely identify an SCB instance, - if two SCB's return the same Id the second SCB is ignored - and subsequent calls to the SCB will only be made to the first - instance. For more information see - Installing an SCB - in the User's Guide. -

- -

This function should NOT have any side effects as it might - be called multiple times by Common Test.

- -

If not implemented the SCB will act as if this function returned a - call to make_ref/0.

- - - - - - - - diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 1a09ec1da2..41371b60be 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -75,7 +75,7 @@ - + diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index 100c0fe5d7..631e3871c2 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -76,7 +76,7 @@ - + diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 81e752680b..7b485bdc35 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -148,8 +148,8 @@ event handlers. ]]>, to install event handlers including start arguments. - ]]>, to install - Suite Callbacks including start arguments. + ]]>, to install + Common Test Hooks including start arguments. , specifies include directories (see above). , disables the automatic test suite compilation feature (see above). ]]>, extends timetrap @@ -444,8 +444,8 @@ {event_handler, EventHandlers, InitArgs}. {event_handler, NodeRefs, EventHandlers, InitArgs}. - {suite_callbacks, SCBModules}. - {suite_callbacks, NodeRefs, SCBModules}. + {ct_hooks, CTHModules}. + {ct_hooks, NodeRefs, CTHModules}.

Test terms:

@@ -484,9 +484,9 @@
       LogDir        = string()
       EventHandlers = atom() | [atom()]
       InitArgs      = [term()]
-      SCBModules    = [SCBModule | {SCBModule, SCBInitArgs}]
-      SCBModule     = atom()
-      SCBInitArgs   = term()
+      CTHModules    = [CTHModule | {CTHModule, CTHInitArgs}]
+      CTHModule     = atom()
+      CTHInitArgs   = term()
       DirRef        = DirAlias | Dir
       Suites        = atom() | [atom()] | all
       Suite         = atom()
diff --git a/lib/common_test/doc/src/suite_callbacks_chapter.xml b/lib/common_test/doc/src/suite_callbacks_chapter.xml
deleted file mode 100644
index 89f78898d4..0000000000
--- a/lib/common_test/doc/src/suite_callbacks_chapter.xml
+++ /dev/null
@@ -1,394 +0,0 @@
-
-
-
-
-  

-    
-      20112011
-      Ericsson AB. All Rights Reserved.
-    
-    
-      The contents of this file are subject to the Erlang Public License,
-      Version 1.1, (the "License"); you may not use this file except in
-      compliance with the License. You should have received a copy of the
-      Erlang Public License along with this software. If not, it can be
-      retrieved online at http://www.erlang.org/.
-
-      Software distributed under the License is distributed on an "AS IS"
-      basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-      the License for the specific language governing rights and limitations
-      under the License.
-
-    
-
-    Suite Callbacks
-    Lukas Larsson
-    
-    
-    
-    suite_callbacks_chapter.xml
-  

-
-  
-  

-    General
-    
This feature is in alpha release right now. This means that the 
-	interface may change in the future and that there may be bugs. We 
-	encourage you to use this feature, but be prepared 
-	that there might be bugs and that the interface might change
-	inbetween releases.

-    

-      The Suite Callback (henceforth called SCB) framework allows 
-      extensions of the default behaviour of Common Test by means of callbacks 
-      before and after all test suite calls. SCBs allow advanced Common Test
-      users to abstract out behaviour which is common to multiple test suites
-      without littering all test suites with library calls. Some example 
-      usages are: logging, starting and monitoring external systems, 
-      building C files needed by the tests and much more!

-
-    
In brief, Suite Callbacks allows you to:

-
-    
-      Manipulate the runtime config before each suite 
-      configuration calls
-      Manipulate the return of all suite configuration calls and in 
-      extension the result of the test themselves.
-    
-    
-    
The following sections describe how to use SCBs, when they are run 
-      and how to manipulate your test results in an SCB

-
-    
When executing within an SCB all timetraps are shutoff. So
-	if your SCB never returns, the entire test run will be stalled!

-    
-
-  

-  
-  
-  

-    Installing an SCB
-    
There are multiple ways to install an SCB in your test run. You can do it
-      for all tests in a run, for specific test suites and for specific groups 
-      within a test suite. If you want an SCB to be present in all test suites 
-      within your testrun there are three different ways to accomplish that.

-
-    
-      Add -suite_callbacks as an argument to 
-      ct_run. 
-      To add multiple SCBs using this method append them to each other
-      using the keyword and, i.e. 
-      ct_run -suite_callbacks scb1 [{debug,true}] and scb2 ....
-      Add the suite_callbacks tag to your 
-      
-      Test Specification
-      Add the suite_callbacks tag to your call to 
-      ct:run_test/1
-    
-
-    
You can also add SCBs within a test suite. This is done by returning
-    {suite_callbacks,[SCB]} in the config list from 
-    suite/0,
-    
-      init_per_suite/1 or
-      
-    init_per_group/2. SCB in this case can be either
-    only the module name of the SCB or a tuple with the module name and the
-    initial arguments to the SCB. Eg:
-    {suite_callbacks,[my_scb_module]} or 
-    {suite_callbacks,[{my_scb_module,[{debug,true}]}]}

-
-    

-      Overriding SCBs
-      
By default each installation of an SCB will cause a new instance of it
-	to be activated. This can cause problems if you want to be able to 
-	override SCBs in testspecifications while still having them in the
-	suite info function. The 
-	id/1
-	callback exists to address this problem. By returning the same
-	id in both places, Common Test knows that this SCB
-	has already been installed and will not try to install it again.

-    

-
-  

-
-  
-  

-    SCB Scope
-    
Once the SCB is installed into a certain test run it will be there until
-      it's scope is expired. The scope of an SCB depends on when it is 
-      installed.
-      The init/2 is 
-      called at the beginning of the scope and the 
-      terminate/1
-     function is called when the scope ends.

-    
-      
-	SCB Installed in
-	SCB scope begins before
-	SCB scope ends after
-      
-      
-	ct_run
-	the first test suite is to be run.
-	the last test suite has been run.
-      
-      
-	ct:run_test
-	the first test suite is to be run.
-	the last test suite has been run.
-      
-      
-	
-	  Test Specification
-	the first test suite is to be run.
-	the last test suite has been run.
-      
-      
-	suite/0
-	
-	
-	    pre_init_per_suite/3 is called.
-	
-	  post_end_per_suite/4 has been called for that test suite.
-      
-      
-	
-	  init_per_suite/1
-	
-	    post_init_per_suite/4 is called.
-	
-	  post_end_per_suite/4 has been called for that test suite.
-      
-      
-	
-	  init_per_group/2
-	
-	    post_init_per_group/4 is called.
-	
-	  post_end_per_group/4 has been called for that group.
-      
-    Scope of an SCB
-    

-    
-    

-      CTH Processes and Tables
-      
CTHs are run with the same process scoping as normal test suites
-	i.e. a different process will execute the init_per_suite hooks then the
-	init_per_group or per_testcase hooks. So if you want to spawn a 
-	process in the CTH you cannot link with the CTH process as it will exit 
-	after the post hook ends. Also if you for some reason need an ETS 
-	table with your CTH, you will have to spawn a process which handles 
-	it.

-    

-    
-  

-
-  
-  

-    Manipulating tests
-    
It is through SCB's possible to manipulate the results of tests and 
-    configuration functions. The main purpose of doing this with SCBs is to
-    allow common patterns to be abstracted out from test test suites and applied to
-    multiple test suites without duplicating any code. All of the callback
-    functions for an SCB follow a common interface, this interface is 
-    described below.

-
-    
-    

-      Pre test manipulation
-      

-	It is possible in an SCB to hook in behaviour before 
-	init_per_suite, 
-	init_per_group, 
-	init_per_testcase, 
-	end_per_group and 
-	end_per_suite. 
-	This is done in the SCB functions called pre_<name of function>.
-	All of these function take the same three arguments: Name, 
-	Config and SCBState. The return value of the SCB function
-	is always a combination of an result for the suite/group/test and an 
-	updated SCBState. If you want the test suite to continue on 
-	executing you should return the config list which you want the test to 
-	use as the result. If you for some reason want to skip/fail the test, 
-	return a tuple with skip or fail and a reason as the 
-	result. Example:
-      

-      pre_init_per_suite(SuiteName, Config, SCBState) ->
-  case db:connect() of
-    {error,_Reason} ->
-      {{fail, "Could not connect to DB"}, SCBState};
-    {ok, Handle} ->
-      {[{db_handle, Handle} | Config], SCBState#state{ handle = Handle }}
-  end.
-	
-    

-    
-    
-    

-      Post test manipulation
-      
It is also possible in an SCB to hook in behaviour after 
-      init_per_suite, 
-      init_per_group, 
-      end_per_testcase, 
-      end_per_group and 
-      end_per_suite.
-      This is done in the SCB functions called post_<name of function>. 
-      All of these function take the same four arguments: Name, 
-      Config, Return and SCBState. Config in this
-      case is the same Config as the testcase is called with. 
-      Return is the value returned by the testcase. If the testcase 
-      failed by crashing, Return will be 
-      {'EXIT',{{Error,Reason},Stacktrace}}.

-      
-      
The return value of the SCB function is always a combination of an
-	result for the suite/group/test and an updated SCBState. If
-	you want the callback to not affect the outcome of the test you should
-	return the Return data as it is given to the SCB. You can also
-	modify the result of the test. By returning the Config list
-	with the tc_status element removed you can recover from a test 
-	failure. As in all the pre hooks, it is also possible to fail/skip
-	the test case in the post hook. Example: 

-
-      post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, SCBState) ->
-  case db:check_consistency() of
-    true ->
-      %% DB is good, pass the test.
-      {proplists:delete(tc_status, Config), SCBState};
-    false ->
-      %% DB is not good, mark as skipped instead of failing
-      {{skip, "DB is inconsisten!"}, SCBState}
-  end;
-post_end_per_testcase(_TC, Config, Return, SCBState) ->
-  %% Do nothing if tc does not crash.
-  {Return, SCBState}.
-
-      Recovering from a testcase failure using SCBs should only be done as
-	a last resort. If used wrongly it could become very difficult to 
-	determine which tests pass or fail in a test run
-  
-    

-
-    
-    

-      Skip and Fail
-      

-	After any post hook has been executed for all installed SCBs, 
-	on_tc_fail
-	or on_tc_skip 
-	might be called if the testcase failed or was skipped 
-	respectively. You cannot affect the outcome of the tests any further at 
-	this point. 
-      

-    

-
-  

-
-  
-  

-     Example SCB
-     
The SCB below will log information about a test run into a format 
-       parseable by file:consult/1.
-     

-     %%% @doc Common Test Example Suite Callback module.
--module(example_scb).
-
-%% Suite Callbacks
--export([id/1]).
--export([init/2]).
-
--export([pre_init_per_suite/3]).
--export([post_init_per_suite/4]).
--export([pre_end_per_suite/3]).
--export([post_end_per_suite/4]).
-
--export([pre_init_per_group/3]).
--export([post_init_per_group/4]).
--export([pre_end_per_group/3]).
--export([post_end_per_group/4]).
-
--export([pre_init_per_testcase/3]).
--export([post_end_per_testcase/4]).
-
--export([on_tc_fail/3]).
--export([on_tc_skip/3]).
-
--export([terminate/1]).
-
--record(state, { file_handle, total, suite_total, ts, tcs, data }).
-
-%% @doc Return a unique id for this SCB.
-id(Opts) ->
-  proplists:get_value(filename, Opts, "/tmp/file.log").
-
-%% @doc Always called before any other callback function. Use this to initiate
-%% any common state. 
-init(Id, Opts) ->
-    {ok,D} = file:open(Id,[write]),
-    #state{ file_handle = D, total = 0, data = [] }.
-
-%% @doc Called before init_per_suite is called. 
-pre_init_per_suite(Suite,Config,State) ->
-    {Config, State#state{ suite_total = 0, tcs = [] }}.
-
-%% @doc Called after init_per_suite.
-post_init_per_suite(Suite,Config,Return,State) ->
-    {Return, State}.
-
-%% @doc Called before end_per_suite. 
-pre_end_per_suite(Suite,Config,State) ->
-    {Config, State}.
-
-%% @doc Called after end_per_suite. 
-post_end_per_suite(Suite,Config,Return,State) ->
-    Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)},
-    {Return, State#state{ data = [Data | State#state.data] ,
-                          total = State#state.total + State#state.suite_total } }.
-
-%% @doc Called before each init_per_group.
-pre_init_per_group(Group,Config,State) ->
-    {Config, State}.
-
-%% @doc Called after each init_per_group.
-post_init_per_group(Group,Config,Return,State) ->
-    {Return, State}.
-
-%% @doc Called after each end_per_group. 
-pre_end_per_group(Group,Config,State) ->
-    {Config, State}.
-
-%% @doc Called after each end_per_group. 
-post_end_per_group(Group,Config,Return,State) ->
-    {Return, State}.
-
-%% @doc Called before each test case.
-pre_init_per_testcase(TC,Config,State) ->
-    {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.
-
-%% @doc Called after each test case.
-post_end_per_testcase(TC,Config,Return,State) ->
-    TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)},
-    {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.
-
-%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
-%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
-on_tc_fail(TC, Reason, State) ->
-    State.
-
-%% @doc Called when a test case is skipped by either user action
-%% or due to an init function failing.  
-on_tc_skip(TC, Reason, State) ->
-    State.
-
-%% @doc Called when the scope of the SCB is done
-terminate(State) ->
-    io:format(State#state.file_handle, "~p.~n",
-               [{test_run, State#state.total, State#state.data}]),
-    file:close(State#state.file_handle),
-    ok.
-  

-
-
-
-
-
-
-- 
cgit v1.2.3