aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--lib/common_test/doc/src/Makefile1
-rw-r--r--lib/common_test/doc/src/common_test_app.xml10
-rw-r--r--lib/common_test/doc/src/ct_suite_callbacks.xml503
-rw-r--r--lib/common_test/doc/src/event_handler_chapter.xml11
-rw-r--r--lib/common_test/doc/src/part.xml1
-rw-r--r--lib/common_test/doc/src/run_test_chapter.xml11
-rw-r--r--lib/common_test/doc/src/suite_callbacks_chapter.xml394
-rw-r--r--lib/common_test/doc/src/write_test_chapter.xml1
-rw-r--r--lib/common_test/src/ct.erl6
9 files changed, 910 insertions, 28 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile
index be066143e0..a3f740852c 100644
--- a/lib/common_test/doc/src/Makefile
+++ b/lib/common_test/doc/src/Makefile
@@ -71,6 +71,7 @@ XML_CHAPTER_FILES = \
cover_chapter.xml \
ct_master_chapter.xml \
event_handler_chapter.xml \
+ suite_callbacks_chapter.xml \
dependencies_chapter.xml \
notes.xml \
notes_history.xml
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml
index e30eef2488..a735dc85a8 100644
--- a/lib/common_test/doc/src/common_test_app.xml
+++ b/lib/common_test/doc/src/common_test_app.xml
@@ -131,7 +131,8 @@
<type>
<v> Info = {timetrap,Time} | {require,Required} |
{require,Name,Required} | {userdata,UserData} |
- {silent_connections,Conns} | {stylesheet,CSSFile}</v>
+ {silent_connections,Conns} | {stylesheet,CSSFile} |
+ {suite_callbacks, SCBs}</v>
<v> Time = MilliSec | {seconds,integer()} | {minutes,integer()}
| {hours,integer()}</v>
<v> MilliSec = integer()</v>
@@ -143,6 +144,9 @@
<v> UserData = term()</v>
<v> Conns = [atom()]</v>
<v> CSSFile = string()</v>
+ <v> SCBs = [SCBModule | {SCBModule, SCBInitArgs}]</v>
+ <v> SCBModule = atom()</v>
+ <v> SCBInitArgs = term()</v>
</type>
<desc>
@@ -170,6 +174,10 @@
<p>With <c>userdata</c>, it is possible for the user to
specify arbitrary test suite related information which can be
read by calling <c>ct:userdata/2</c>.</p>
+
+ <p>The <c>suite_callbacks</c> tag specifies which
+ <seealso marker="suite_callbacks_chapter">Suite Callbacks</seealso>
+ are to be run together with this suite.</p>
<p>Other tuples than the ones defined will simply be ignored.</p>
diff --git a/lib/common_test/doc/src/ct_suite_callbacks.xml b/lib/common_test/doc/src/ct_suite_callbacks.xml
index 28a496a47b..cea5804825 100644
--- a/lib/common_test/doc/src/ct_suite_callbacks.xml
+++ b/lib/common_test/doc/src/ct_suite_callbacks.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="latin1" ?>
+<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE erlref SYSTEM "erlref.dtd">
@@ -33,24 +33,43 @@
<file>suite_callback.sgml</file>
</header>
<module>ct_suite_callback</module>
- <modulesummary>A callback interface on top of common test</modulesummary>
+ <modulesummary>A callback interface on top of Common Test</modulesummary>
<description>
- <p>The <em>Suite Callback</em> framework allows extensions of the default
- behaviour of Common Test by means of callbacks before and after all
- test suite calls. </p>
+ <warning><p>This feature is in alpha release right now. This means that the
+ interface may change in the future and that there may be bugs. We
+ encourage you to use this feature, but be prepared
+ that there might be bugs and that the interface might change
+ inbetween releases.</p></warning>
+
+ <p>The <em>Suite Callback</em> (henceforth called SCB) framework allows
+ extensions of the default behaviour of Common Test by means of callbacks
+ before and after all test suite calls. It is meant for advanced users of
+ Common Test which want to abstract out behaviour which is common to
+ multiple test suites. </p>
<p>In brief, Suite Callbacks allows you to:</p>
<list>
- <item>Manipulating the runtime config before each suite configuration calls</item>
- <item>Manipulating the return of all suite configuration calls</item>
+ <item>Manipulate the runtime config before each suite
+ configuration calls</item>
+ <item>Manipulate the return of all suite configuration calls and in
+ extension the result of the test themselves.</item>
</list>
- <p>The following sections describe the mandatory and optional suite callback
- functions Common Test will call during test execution. For more details
- see <seealso marker="suite_callbacks">Suite Callbacks</seealso> in the User's Guide.</p>
+ <p>The following sections describe the mandatory and optional SCB
+ functions Common Test will call during test execution. For more details
+ see <seealso marker="suite_callbacks_chapter">Suite Callbacks</seealso> in
+ the User's Guide.</p>
+
+ <p>For information about how to add a SCB to your suite see
+ <seealso marker="suite_callbacks_chapter#installing">Installing an SCB
+ </seealso> in the User's Guide.</p>
+
+ <note><p>See the
+ <seealso marker="suite_callbacks_chapter#example">Example SCB</seealso>
+ in the User's Guide for a minimal example of an SCB. </p></note>
</description>
@@ -62,31 +81,469 @@
<funcs>
<func>
- <name>Module:init(Opts) -> {Id,State} </name>
- <fsummary>asdas</fsummary>
+ <name>Module:init(Id, Opts) -&gt; State</name>
+ <fsummary>Initiates the Suite Callback</fsummary>
<type>
+ <v>Id = reference() | term()</v>
<v>Opts = term()</v>
- <v>Id = term()</v>
- <v>GroupName = term()</v>
+ <v>State = term()</v>
</type>
<desc>
<p> MANDATORY </p>
+
+ <p>Always called before any other callback function.
+ Use this to initiate any common state.
+ It should return a state for this SCB.</p>
+
+ <p><c>Id</c> is the return value of
+ <seealso marker="#Module:id-1">id/1</seealso>, or a <c>reference</c>
+ if <seealso marker="#Module:id-1">id/1</seealso> is not implemented.
+ </p>
+
+ <p>For details about when init is called see
+ <seealso marker="suite_callbacks_chapter#scope">scope</seealso>
+ in the User's Guide.</p>
+
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:pre_init_per_suite(SuiteName, Config, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called before init_per_suite</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>Config = NewConfig = [{Key,Value}]</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {Return, NewSCBState}</v>
+ <v>Return = NewConfig | SkipOrFail</v>
+ <v>SkipOrFail = {fail, Reason} | {skip, Reason}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called before
+ <seealso marker="common_test#Module:init_per_suite-1">
+ init_per_suite</seealso> if it exists.
+ It typically contains initialization/logging which needs to be done
+ before init_per_suite is called.
+ If <c>{skip,Reason}</c> or <c>{fail,Reason}</c> is returned,
+ init_per_suite and all test cases of the suite will be skipped and
+ Reason printed in the overview log of the suite.</p>
- <p></p>
+ <p><c>SuiteName</c> is the name of the suite to be run.</p>
+
+ <p><c>Config</c> is the original config list of the test suite.</p>
+
+ <p><c>SCBState</c> is the current internal state of the SCB.</p>
+
+ <p><c>Return</c> is the result of the init_per_suite function.
+ If it is <c>{skip,Reason}</c> or <c>{fail,Reason}</c>
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite
+ </seealso> will never be called, instead the initiation is considered
+ to be skipped/failed respectively. If a <c>NewConfig</c> list
+ is returned, <seealso marker="common_test#Module:init_per_suite-1">
+ init_per_suite</seealso> will be called with that <c>NewConfig</c> list.
+ See <seealso marker="suite_callbacks_chapter#manipulating">
+ Manipulating tests</seealso> in the User's Guide for more details.</p>
+
+
+ <p>Note that this function is only called if the SCB has been added
+ before init_per_suite is run, see
+ <seealso marker="suite_callbacks_chapter#scope">SCB Scoping</seealso>
+ in the User's Guide for details.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:post_init_per_suite(SuiteName, Config, Return, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called after init_per_suite</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>Config = [{Key,Value}]</v>
+ <v>Return = NewReturn = Config | SkipOrFail | term()</v>
+ <v>SkipOrFail = {fail, Reason} | {skip, Reason} | term()</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewReturn, NewSCBState}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p><c>Return</c> is what
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite
+ </seealso> returned, i.e. {fail,Reason}, {skip,Reason}, a <c>Config</c>
+ list or a term describing how
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite
+ </seealso> failed.</p>
+
+ <p><c>NewReturn</c> is the possibly modified return value of
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite
+ </seealso>. It is here possible to recover from a failure in
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite
+ </seealso> by returning the <c>ConfigList</c> with the <c>tc_status</c>
+ element removed.</p>
+
+ <p><c>SCBState</c> is the current internal state of the SCB.</p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:init_per_suite-1">
+ init_per_suite</seealso> if it exists.</p>
+
+ <p>Note that this function is only called if the SCB has been added
+ before or in init_per_suite, see
+ <seealso marker="suite_callbacks_chapter#scope">SCB Scoping</seealso>
+ in the User's Guide for details.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:pre_init_per_group(GroupName, Config, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called before init_per_group</fsummary>
+ <type>
+ <v>GroupName = atom()</v>
+ <v>Config = NewConfig = [{Key,Value}]</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
- <p> If <c>{skip,Reason}</c> is returned, all test cases
- in the module will be skipped, and the <c>Reason</c> will
- be printed on the HTML result page.</p>
+ <p>This function is called before
+ <seealso marker="common_test#Module:init_per_group-2">
+ init_per_group</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3">
+ pre_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:init_per_group-2">
+ init_per_group</seealso> instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:post_init_per_group(GroupName, Config, Return, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called after init_per_group</fsummary>
+ <type>
+ <v>GroupName = atom()</v>
+ <v>Config = [{Key,Value}]</v>
+ <v>Return = NewReturn = Config | SkipOrFail | term()</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewReturn, NewSCBState}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
- <p>For details on groups, see
- <seealso marker="write_test_chapter#test_case_groups">Test case
- groups</seealso> in the User's Guide.</p>
+ <p>This function is called after
+ <seealso marker="common_test#Module:init_per_group-2">
+ init_per_group</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:post_init_per_suite-4">
+ post_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:init_per_group-2">
+ init_per_group</seealso> instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:pre_init_per_testcase(TestcaseName, Config, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called before init_per_testcase</fsummary>
+ <type>
+ <v>TestcaseName = atom()</v>
+ <v>Config = NewConfig = [{Key,Value}]</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
- </desc>
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called before
+ <seealso marker="common_test#Module:init_per_testcase-2">
+ init_per_testcase</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3">
+ pre_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:init_per_testcase-2">
+ init_per_testcase</seealso> function instead.</p>
+
+ <p>Note that it is not possible to add SCB's here right now,
+ that feature might be added later,
+ but it would right now break backwards compatability.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:post_end_per_testcase(TestcaseName, Config, Return, SCBState)
+ -&gt; Result</name>
+ <fsummary>Called after end_per_testcase</fsummary>
+ <type>
+ <v>TestcaseName = atom()</v>
+ <v>Config = [{Key,Value}]</v>
+ <v>Return = NewReturn = Config | SkipOrFail | term()</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewReturn, NewSCBState}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:end_per_testcase-2">
+ end_per_testcase</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:post_init_per_suite-4">
+ post_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:end_per_testcase-2">
+ end_per_testcase</seealso> function instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:pre_end_per_group(GroupName, Config, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called before end_per_group</fsummary>
+ <type>
+ <v>GroupName = atom()</v>
+ <v>Config = NewConfig = [{Key,Value}]</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called before
+ <seealso marker="common_test#Module:end_per_group-2">
+ end_per_group</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3">
+ pre_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:end_per_group-2">
+ end_per_group</seealso> function instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:post_end_per_group(GroupName, Config, Return, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called after end_per_group</fsummary>
+ <type>
+ <v>GroupName = atom()</v>
+ <v>Config = [{Key,Value}]</v>
+ <v>Return = NewReturn = Config | SkipOrFail | term()</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewReturn, NewSCBState}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:end_per_group-2">
+ end_per_group</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:post_init_per_suite-4">
+ post_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:end_per_group-2">
+ end_per_group</seealso> function instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:pre_end_per_suite(SuiteName, Config, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called before end_per_suite</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>Config = NewConfig = [{Key,Value}]</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called before
+ <seealso marker="common_test#Module:end_per_suite-1">
+ end_per_suite</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3">
+ pre_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:end_per_suite-2">
+ end_per_suite</seealso> function instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:post_end_per_suite(SuiteName, Config, Return, SCBState) -&gt;
+ Result</name>
+ <fsummary>Called after end_per_suite</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>Config = [{Key,Value}]</v>
+ <v>Return = NewReturn = Config | SkipOrFail | term()</v>
+ <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v>
+ <v>SCBState = NewSCBState = term()</v>
+ <v>Result = {NewReturn, NewSCBState}</v>
+ <v>Key = atom()</v>
+ <v>Value = term()</v>
+ <v>Reason = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:end_per_suite-1">
+ end_per_suite</seealso> if it exists. It behaves the same way as
+ <seealso marker="ct_suite_callback#Module:post_init_per_suite-4">
+ post_init_per_suite</seealso>, but for the
+ <seealso marker="common_test#Module:end_per_suite-2">
+ end_per_suite</seealso> function instead.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:on_tc_fail(TestcaseName, Reason, SCBState) -&gt;
+ NewSCBState</name>
+ <fsummary>Called after the SCB scope ends</fsummary>
+ <type>
+ <v>TestcaseName = init_per_suite | end_per_suite |
+ init_per_group | end_per_group | atom()</v>
+ <v>Reason = term()</v>
+ <v>SCBState = NewSCBState = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called whenever a testcase fails.
+ It is called after the post function has been called for
+ the testcase which failed. i.e.
+ if init_per_suite fails this function is called after
+ <seealso marker="#Module:post_init_per_suite-4">
+ post_init_per_suite</seealso>, and if a testcase fails it is called
+ after <seealso marker="#Module:post_end_per_testcase-4">
+ post_end_per_testcase</seealso>.</p>
+
+ <p>The data which comes with the Reason follows the same format as the
+ <seealso marker="event_handler_chapter#failreason">FailReason
+ </seealso> in the <seealso marker="event_handler_chapter#tc_done">tc_done</seealso> event.
+ See <seealso marker="event_handler_chapter#events">Event Handling
+ </seealso> in the User's Guide for details.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:on_tc_skip(TestcaseName, Reason, SCBState) -&gt;
+ NewSCBState</name>
+ <fsummary>Called after the SCB scope ends</fsummary>
+ <type>
+ <v>TestcaseName = end_per_suite | init_per_group |
+ end_per_group | atom()</v>
+ <v>Reason = {tc_auto_skip | tc_user_skip, term()}</v>
+ <v>SCBState = NewSCBState = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called whenever a testcase is skipped.
+ It is called after the post function has been called for the
+ testcase which was skipped.
+ i.e. if init_per_group is skipped this function is called after
+ <seealso marker="#Module:post_init_per_suite-4">post_init_per_group
+ </seealso>, and if a testcase is skipped it is called after
+ <seealso marker="#Module:post_end_per_testcase-4">post_end_per_testcase
+ </seealso>.</p>
+
+ <p>The data which comes with the Reason follows the same format as
+ <seealso marker="event_handler_chapter#tc_auto_skip">tc_auto_skip
+ </seealso> and <seealso marker="event_handler_chapter#tc_user_skip">
+ tc_user_skip</seealso> events.
+ See <seealso marker="event_handler_chapter#events">Event Handling
+ </seealso> in the User's Guide for details.</p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:terminate(SCBState)</name>
+ <fsummary>Called after the SCB scope ends</fsummary>
+ <type>
+ <v>SCBState = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>This function is called at the end of an SCB's
+ <seealso marker="suite_callbacks_chapter#scope">scope</seealso>.
+ </p>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:id(Opts) -&gt; Id</name>
+ <fsummary>Called before the init function of an SCB</fsummary>
+ <type>
+ <v>Opts = term()</v>
+ <v>Id = term()</v>
+ </type>
+
+ <desc>
+ <p> OPTIONAL </p>
+
+ <p>The <c>Id</c> is used to uniquely identify an SCB instance,
+ if two SCB's return the same <c>Id</c> the second SCB is ignored
+ and subsequent calls to the SCB will only be made to the first
+ instance. For more information see
+ <seealso marker="suite_callbacks_chapter#installing">Installing an SCB
+ </seealso> in the User's Guide.
+ </p>
+
+ <p>This function should NOT have any side effects as it might
+ be called multiple times by Common Test.</p>
+
+ <p>If not implemented the SCB will act as if this function returned a
+ call to <c>make_ref/0</c>.</p>
+ </desc>
</func>
-
</funcs>
</erlref>
diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml
index 904876ac46..a01feb59d1 100644
--- a/lib/common_test/doc/src/event_handler_chapter.xml
+++ b/lib/common_test/doc/src/event_handler_chapter.xml
@@ -61,6 +61,7 @@
itself.</p>
</section>
<section>
+ <marker id="usage"></marker>
<title>Usage</title>
<p>Event handlers may be installed by means of an <c>event_handler</c>
start flag (<c>ct_run</c>) or option (<c>ct:run_test/1</c>), where the
@@ -120,6 +121,7 @@
node the event has originated from (only relevant for CT Master event handlers).
<c>data</c> is specific for the particular event.</p>
+ <marker id="events"></marker>
<p><em>General events:</em></p>
<list>
@@ -172,6 +174,7 @@
are also given.
</p></item>
+ <marker id="tc_done"/>
<item><c>#event{name = tc_done, data = {Suite,FuncOrGroup,Result}}</c>
<p><c>Suite = atom()</c>, name of the suite.</p>
<p><c>FuncOrGroup = Func | {Conf,GroupName,GroupProperties}</c></p>
@@ -181,12 +184,14 @@
(unknown if init- or end function times out).</p>
<p><c>GroupProperties = list()</c>, list of execution properties for the group.</p>
<p><c>Result = ok | {skipped,SkipReason} | {failed,FailReason}</c>, the result.</p>
+ <marker id="skipreason"/>
<p><c>SkipReason = {require_failed,RequireInfo} |
{require_failed_in_suite0,RequireInfo} |
{failed,{Suite,init_per_testcase,FailInfo}} |
UserTerm</c>,
the reason why the case has been skipped.</p>
- <p><c>FailReason = {error,FailInfo} |
+ <marker id="failreason"/>
+ <p><c>FailReason = {error,FailInfo} |
{error,{RunTimeError,StackTrace}} |
{timetrap_timeout,integer()} |
{failed,{Suite,end_per_testcase,FailInfo}}</c>, reason for failure.</p>
@@ -209,6 +214,7 @@
<c>end_per_testcase</c> for the case failed.
</p></item>
+ <marker id="tc_auto_skip"></marker>
<item><c>#event{name = tc_auto_skip, data = {Suite,Func,Reason}}</c>
<p><c>Suite = atom()</c>, the name of the suite.</p>
<p><c>Func = atom()</c>, the name of the test case or configuration function.</p>
@@ -234,7 +240,8 @@
skipped because of <c>init_per_testcase</c> failing, since that information is carried with
the <c>tc_done</c> event.
</p></item>
-
+
+ <marker id="tc_user_skip"></marker>
<item><c>#event{name = tc_user_skip, data = {Suite,TestCase,Comment}}</c>
<p><c>Suite = atom()</c>, name of the suite.</p>
<p><c>TestCase = atom()</c>, name of the test case.</p>
diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml
index 53a4cb1bbf..1a09ec1da2 100644
--- a/lib/common_test/doc/src/part.xml
+++ b/lib/common_test/doc/src/part.xml
@@ -75,6 +75,7 @@
<xi:include href="ct_master_chapter.xml"/>
<xi:include href="event_handler_chapter.xml"/>
<xi:include href="dependencies_chapter.xml"/>
+ <xi:include href="suite_callbacks_chapter.xml"/>
<xi:include href="why_test_chapter.xml"/>
</part>
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 94fcf6bf01..81e752680b 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -105,6 +105,7 @@
RPC from a remote node.</p>
</section>
+ <marker id="ct_run"></marker>
<section>
<title>Running tests from the OS command line</title>
@@ -147,6 +148,8 @@
<seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item>
<item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install
<seealso marker="event_handler_chapter#event_handling">event handlers</seealso> including start arguments.</item>
+ <item><c><![CDATA[-suite_callbacks <suite_callbacks>]]></c>, to install
+ <seealso marker="suite_callbacks_chapter#installing">Suite Callbacks</seealso> including start arguments.</item>
<item><c><![CDATA[-include]]></c>, specifies include directories (see above).</item>
<item><c><![CDATA[-no_auto_compile]]></c>, disables the automatic test suite compilation feature (see above).</item>
<item><c><![CDATA[-multiply_timetraps <n>]]></c>, extends <seealso marker="write_test_chapter#timetraps">timetrap
@@ -333,8 +336,8 @@
with <c>dir</c>.</p>
</section>
+ <marker id="test_specifications"></marker>
<section>
- <marker id="test_specifications"></marker>
<title>Using test specifications</title>
<p>The most flexible way to specify what to test, is to use a so
@@ -440,6 +443,9 @@
{event_handler, NodeRefs, EventHandlers}.
{event_handler, EventHandlers, InitArgs}.
{event_handler, NodeRefs, EventHandlers, InitArgs}.
+
+ {suite_callbacks, SCBModules}.
+ {suite_callbacks, NodeRefs, SCBModules}.
</pre>
<p>Test terms:</p>
<pre>
@@ -478,6 +484,9 @@
LogDir = string()
EventHandlers = atom() | [atom()]
InitArgs = [term()]
+ SCBModules = [SCBModule | {SCBModule, SCBInitArgs}]
+ SCBModule = atom()
+ SCBInitArgs = term()
DirRef = DirAlias | Dir
Suites = atom() | [atom()] | all
Suite = atom()
diff --git a/lib/common_test/doc/src/suite_callbacks_chapter.xml b/lib/common_test/doc/src/suite_callbacks_chapter.xml
new file mode 100644
index 0000000000..89f78898d4
--- /dev/null
+++ b/lib/common_test/doc/src/suite_callbacks_chapter.xml
@@ -0,0 +1,394 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE chapter SYSTEM "chapter.dtd">
+
+<chapter>
+ <header>
+ <copyright>
+ <year>2011</year><year>2011</year>
+ <holder>Ericsson AB. All Rights Reserved.</holder>
+ </copyright>
+ <legalnotice>
+ The contents of this file are subject to the Erlang Public License,
+ Version 1.1, (the "License"); you may not use this file except in
+ compliance with the License. You should have received a copy of the
+ Erlang Public License along with this software. If not, it can be
+ retrieved online at http://www.erlang.org/.
+
+ Software distributed under the License is distributed on an "AS IS"
+ basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ the License for the specific language governing rights and limitations
+ under the License.
+
+ </legalnotice>
+
+ <title>Suite Callbacks</title>
+ <prepared>Lukas Larsson</prepared>
+ <docno></docno>
+ <date></date>
+ <rev></rev>
+ <file>suite_callbacks_chapter.xml</file>
+ </header>
+
+ <marker id="general"></marker>
+ <section>
+ <title>General</title>
+ <warning><p>This feature is in alpha release right now. This means that the
+ interface may change in the future and that there may be bugs. We
+ encourage you to use this feature, but be prepared
+ that there might be bugs and that the interface might change
+ inbetween releases.</p></warning>
+ <p>
+ The <em>Suite Callback</em> (henceforth called SCB) framework allows
+ extensions of the default behaviour of Common Test by means of callbacks
+ before and after all test suite calls. SCBs allow advanced Common Test
+ users to abstract out behaviour which is common to multiple test suites
+ without littering all test suites with library calls. Some example
+ usages are: logging, starting and monitoring external systems,
+ building C files needed by the tests and much more!</p>
+
+ <p>In brief, Suite Callbacks allows you to:</p>
+
+ <list>
+ <item>Manipulate the runtime config before each suite
+ configuration calls</item>
+ <item>Manipulate the return of all suite configuration calls and in
+ extension the result of the test themselves.</item>
+ </list>
+
+ <p>The following sections describe how to use SCBs, when they are run
+ and how to manipulate your test results in an SCB</p>
+
+ <warning><p>When executing within an SCB all timetraps are shutoff. So
+ if your SCB never returns, the entire test run will be stalled!</p>
+ </warning>
+
+ </section>
+
+ <marker id="installing"></marker>
+ <section>
+ <title>Installing an SCB</title>
+ <p>There are multiple ways to install an SCB in your test run. You can do it
+ for all tests in a run, for specific test suites and for specific groups
+ within a test suite. If you want an SCB to be present in all test suites
+ within your testrun there are three different ways to accomplish that.</p>
+
+ <list>
+ <item>Add <c>-suite_callbacks</c> as an argument to
+ <seealso marker="run_test_chapter#ct_run">ct_run</seealso>.
+ To add multiple SCBs using this method append them to each other
+ using the keyword <c>and</c>, i.e.
+ <c>ct_run -suite_callbacks scb1 [{debug,true}] and scb2 ...</c>.</item>
+ <item>Add the <c>suite_callbacks</c> tag to your
+ <seealso marker="run_test_chapter#test_specifications">
+ Test Specification</seealso></item>
+ <item>Add the <c>suite_callbacks</c> tag to your call to
+ <seealso marker="ct#run_test-1">ct:run_test/1</seealso></item>
+ </list>
+
+ <p>You can also add SCBs within a test suite. This is done by returning
+ <c>{suite_callbacks,[SCB]}</c> in the config list from
+ <seealso marker="common_test#Module:suite-0">suite/0</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">
+ init_per_suite/1</seealso> or
+ <seealso marker="common_test#Module:init_per_group-2">
+ init_per_group/2</seealso>. <c>SCB</c> in this case can be either
+ only the module name of the SCB or a tuple with the module name and the
+ initial arguments to the SCB. Eg:
+ <c>{suite_callbacks,[my_scb_module]}</c> or
+ <c>{suite_callbacks,[{my_scb_module,[{debug,true}]}]}</c></p>
+
+ <section>
+ <title>Overriding SCBs</title>
+ <p>By default each installation of an SCB will cause a new instance of it
+ to be activated. This can cause problems if you want to be able to
+ override SCBs in testspecifications while still having them in the
+ suite info function. The
+ <seealso marker="ct_suite_callback#Module:id-1">id/1</seealso>
+ callback exists to address this problem. By returning the same
+ <c>id</c> in both places, Common Test knows that this SCB
+ has already been installed and will not try to install it again.</p>
+ </section>
+
+ </section>
+
+ <marker id="scope"/>
+ <section>
+ <title>SCB Scope</title>
+ <p>Once the SCB is installed into a certain test run it will be there until
+ it's scope is expired. The scope of an SCB depends on when it is
+ installed.
+ The <seealso marker="ct_suite_callback#Module:init-2">init/2</seealso> is
+ called at the beginning of the scope and the
+ <seealso marker="ct_suite_callback#Module:terminate-1">terminate/1
+ </seealso> function is called when the scope ends.</p>
+ <table>
+ <row>
+ <cell><em>SCB Installed in</em></cell>
+ <cell><em>SCB scope begins before</em></cell>
+ <cell><em>SCB scope ends after</em></cell>
+ </row>
+ <row>
+ <cell><seealso marker="run_test_chapter#ct_run">ct_run</seealso></cell>
+ <cell>the first test suite is to be run.</cell>
+ <cell>the last test suite has been run.</cell>
+ </row>
+ <row>
+ <cell><seealso marker="ct#run_test-1">ct:run_test</seealso></cell>
+ <cell>the first test suite is to be run.</cell>
+ <cell>the last test suite has been run.</cell>
+ </row>
+ <row>
+ <cell><seealso marker="run_test_chapter#test_specifications">
+ Test Specification</seealso></cell>
+ <cell>the first test suite is to be run.</cell>
+ <cell>the last test suite has been run.</cell>
+ </row>
+ <row>
+ <cell><seealso marker="common_test#Module:suite-0">suite/0
+ </seealso></cell>
+ <cell><seealso marker="ct_suite_callback#Module:pre_init_per_suite-3">
+ pre_init_per_suite/3</seealso> is called.</cell>
+ <cell><seealso marker="ct_suite_callback#Module:post_end_per_suite-4">
+ post_end_per_suite/4</seealso> has been called for that test suite.</cell>
+ </row>
+ <row>
+ <cell><seealso marker="common_test#Module:init_per_suite-1">
+ init_per_suite/1</seealso></cell>
+ <cell><seealso marker="ct_suite_callback#Module:post_init_per_suite-4">
+ post_init_per_suite/4</seealso> is called.</cell>
+ <cell><seealso marker="ct_suite_callback#Module:post_end_per_suite-4">
+ post_end_per_suite/4</seealso> has been called for that test suite.</cell>
+ </row>
+ <row>
+ <cell><seealso marker="common_test#Module:init_per_group-2">
+ init_per_group/2</seealso></cell>
+ <cell><seealso marker="ct_suite_callback#Module:post_init_per_group-4">
+ post_init_per_group/4</seealso> is called.</cell>
+ <cell><seealso marker="ct_suite_callback#Module:post_end_per_suite-4">
+ post_end_per_group/4</seealso> has been called for that group.</cell>
+ </row>
+ <tcaption>Scope of an SCB</tcaption>
+ </table>
+
+ <section>
+ <title>CTH Processes and Tables</title>
+ <p>CTHs are run with the same process scoping as normal test suites
+ i.e. a different process will execute the init_per_suite hooks then the
+ init_per_group or per_testcase hooks. So if you want to spawn a
+ process in the CTH you cannot link with the CTH process as it will exit
+ after the post hook ends. Also if you for some reason need an ETS
+ table with your CTH, you will have to spawn a process which handles
+ it.</p>
+ </section>
+
+ </section>
+
+ <marker id="manipulating"/>
+ <section>
+ <title>Manipulating tests</title>
+ <p>It is through SCB's possible to manipulate the results of tests and
+ configuration functions. The main purpose of doing this with SCBs is to
+ allow common patterns to be abstracted out from test test suites and applied to
+ multiple test suites without duplicating any code. All of the callback
+ functions for an SCB follow a common interface, this interface is
+ described below.</p>
+
+ <marker id="pre"/>
+ <section>
+ <title>Pre test manipulation</title>
+ <p>
+ It is possible in an SCB to hook in behaviour before
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_testcase</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and
+ <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>.
+ This is done in the SCB functions called pre_&lt;name of function&gt;.
+ All of these function take the same three arguments: <c>Name</c>,
+ <c>Config</c> and <c>SCBState</c>. The return value of the SCB function
+ is always a combination of an result for the suite/group/test and an
+ updated <c>SCBState</c>. If you want the test suite to continue on
+ executing you should return the config list which you want the test to
+ use as the result. If you for some reason want to skip/fail the test,
+ return a tuple with <c>skip</c> or <c>fail</c> and a reason as the
+ result. Example:
+ </p>
+ <code>pre_init_per_suite(SuiteName, Config, SCBState) -&gt;
+ case db:connect() of
+ {error,_Reason} -&gt;
+ {{fail, "Could not connect to DB"}, SCBState};
+ {ok, Handle} -&gt;
+ {[{db_handle, Handle} | Config], SCBState#state{ handle = Handle }}
+ end.</code>
+
+ </section>
+
+ <marker id="post"/>
+ <section>
+ <title>Post test manipulation</title>
+ <p>It is also possible in an SCB to hook in behaviour after
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">end_per_testcase</seealso>,
+ <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and
+ <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>.
+ This is done in the SCB functions called post_&lt;name of function&gt;.
+ All of these function take the same four arguments: <c>Name</c>,
+ <c>Config</c>, <c>Return</c> and <c>SCBState</c>. <c>Config</c> in this
+ case is the same <c>Config</c> as the testcase is called with.
+ <c>Return</c> is the value returned by the testcase. If the testcase
+ failed by crashing, <c>Return</c> will be
+ <c>{'EXIT',{{Error,Reason},Stacktrace}}</c>.</p>
+
+ <p>The return value of the SCB function is always a combination of an
+ result for the suite/group/test and an updated <c>SCBState</c>. If
+ you want the callback to not affect the outcome of the test you should
+ return the <c>Return</c> data as it is given to the SCB. You can also
+ modify the result of the test. By returning the <c>Config</c> list
+ with the <c>tc_status</c> element removed you can recover from a test
+ failure. As in all the pre hooks, it is also possible to fail/skip
+ the test case in the post hook. Example: </p>
+
+ <code>post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, SCBState) -&gt;
+ case db:check_consistency() of
+ true ->
+ %% DB is good, pass the test.
+ {proplists:delete(tc_status, Config), SCBState};
+ false ->
+ %% DB is not good, mark as skipped instead of failing
+ {{skip, "DB is inconsisten!"}, SCBState}
+ end;
+post_end_per_testcase(_TC, Config, Return, SCBState) -&gt;
+ %% Do nothing if tc does not crash.
+ {Return, SCBState}.</code>
+
+ <note>Recovering from a testcase failure using SCBs should only be done as
+ a last resort. If used wrongly it could become very difficult to
+ determine which tests pass or fail in a test run</note>
+
+ </section>
+
+ <marker id="skip_n_fail"/>
+ <section>
+ <title>Skip and Fail</title>
+ <p>
+ After any post hook has been executed for all installed SCBs,
+ <seealso marker="ct_suite_callback#Module:on_tc_fail-3">on_tc_fail</seealso>
+ or <seealso marker="ct_suite_callback#Module:on_tc_fail-3">on_tc_skip</seealso>
+ might be called if the testcase failed or was skipped
+ respectively. You cannot affect the outcome of the tests any further at
+ this point.
+ </p>
+ </section>
+
+ </section>
+
+ <marker id="example"/>
+ <section>
+ <title>Example SCB</title>
+ <p>The SCB below will log information about a test run into a format
+ parseable by <seealso marker="kernel:file#consult-1">file:consult/1</seealso>.
+ </p>
+ <code>%%% @doc Common Test Example Suite Callback module.
+-module(example_scb).
+
+%% Suite Callbacks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-record(state, { file_handle, total, suite_total, ts, tcs, data }).
+
+%% @doc Return a unique id for this SCB.
+id(Opts) ->
+ proplists:get_value(filename, Opts, "/tmp/file.log").
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state.
+init(Id, Opts) ->
+ {ok,D} = file:open(Id,[write]),
+ #state{ file_handle = D, total = 0, data = [] }.
+
+%% @doc Called before init_per_suite is called.
+pre_init_per_suite(Suite,Config,State) ->
+ {Config, State#state{ suite_total = 0, tcs = [] }}.
+
+%% @doc Called after init_per_suite.
+post_init_per_suite(Suite,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before end_per_suite.
+pre_end_per_suite(Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after end_per_suite.
+post_end_per_suite(Suite,Config,Return,State) ->
+ Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)},
+ {Return, State#state{ data = [Data | State#state.data] ,
+ total = State#state.total + State#state.suite_total } }.
+
+%% @doc Called before each init_per_group.
+pre_init_per_group(Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each init_per_group.
+post_init_per_group(Group,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called after each end_per_group.
+pre_end_per_group(Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each end_per_group.
+post_end_per_group(Group,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before each test case.
+pre_init_per_testcase(TC,Config,State) ->
+ {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.
+
+%% @doc Called after each test case.
+post_end_per_testcase(TC,Config,Return,State) ->
+ TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)},
+ {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
+on_tc_fail(TC, Reason, State) ->
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing.
+on_tc_skip(TC, Reason, State) ->
+ State.
+
+%% @doc Called when the scope of the SCB is done
+terminate(State) ->
+ io:format(State#state.file_handle, "~p.~n",
+ [{test_run, State#state.total, State#state.data}]),
+ file:close(State#state.file_handle),
+ ok.</code>
+ </section>
+
+</chapter>
+
+
+
+
diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml
index 5afec6de6a..76493d3616 100644
--- a/lib/common_test/doc/src/write_test_chapter.xml
+++ b/lib/common_test/doc/src/write_test_chapter.xml
@@ -115,6 +115,7 @@
</p>
</section>
+ <marker id="per_testcase"/>
<section>
<title>Init and end per test case</title>
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 405dc40c8b..eb0eceeb46 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -148,7 +148,8 @@ run(TestDirs) ->
%%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} |
%%% {repeat,N} | {duration,DurTime} | {until,StopTime} |
%%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
-%%% {refresh_logs,LogDir} | {basic_html,Bool}
+%%% {refresh_logs,LogDir} | {basic_html,Bool} |
+%%% {suite_callbacks, SCBs}
%%% TestDirs = [string()] | string()
%%% Suites = [string()] | string()
%%% Cases = [atom()] | atom()
@@ -176,6 +177,9 @@ run(TestDirs) ->
%%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile}
%%% DecryptKey = string()
%%% DecryptFile = string()
+%%% SCBs = [SCBModule | {SCBModule, SCBInitArgs}]
+%%% SCBModule = atom()
+%%% SCBInitArgs = term()
%%% Result = [TestResult] | {error,Reason}
%%% @doc Run tests as specified by the combination of options in <code>Opts</code>.
%%% The options are the same as those used with the