aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/doc
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2017-01-25 11:24:58 +0100
committerSiri Hansen <[email protected]>2017-02-20 12:40:08 +0100
commita95a22017523229bb8924afbd928d3e16b564fa5 (patch)
treeda620b3a4b0e58bbabb5881556e4043cafceb1c2 /lib/common_test/doc
parentd64fa0f13cd07b48291d1ab3b843c411aaf8451d (diff)
downloadotp-a95a22017523229bb8924afbd928d3e16b564fa5.tar.gz
otp-a95a22017523229bb8924afbd928d3e16b564fa5.tar.bz2
otp-a95a22017523229bb8924afbd928d3e16b564fa5.zip
[ct] Add Suite argument to hook callback functions
An extra argument, Suite, is added as the first argument to each of the following hook callback functions: - pre_init_per_group - post_init_per_group - pre_end_per_group - post_end_per_group - pre_init_per_testcase - post_init_per_testcase - pre_end_per_testcase - post_end_per_testcase - on_tc_fail - on_tc_skip For backwards compatibility, if the new function is not exported from a hook callback module, common_test will fall back to the old interface and call the function without the Suite argument. The reason for adding the new argument is that if a test suite is skipped by a 'skip_suites' statement in the test specification, then there will be no call to pre/post_init_per_suite, and thus the hook has no other way of knowing which Suite is skipped when it gets the on_tc_skip callback. The other callbacks are updated for symmetry.
Diffstat (limited to 'lib/common_test/doc')
-rw-r--r--lib/common_test/doc/src/ct_hooks.xml90
-rw-r--r--lib/common_test/doc/src/ct_hooks_chapter.xml74
2 files changed, 113 insertions, 51 deletions
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index c2cf29c530..a085f30262 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -208,9 +208,10 @@
</func>
<func>
- <name>Module:pre_init_per_group(GroupName, InitData, CTHState) -&gt; Result</name>
+ <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -&gt; Result</name>
<fsummary>Called before init_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>InitData = Config | SkipOrFail</v>
<v>Config = NewConfig = [{Key,Value}]</v>
@@ -231,13 +232,19 @@
but for function
<seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>
instead.</p>
+
+ <p>If <c>Module:pre_init_per_group/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_init_per_group(GroupName,
+ InitData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after init_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -258,13 +265,19 @@
but for function
<seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>
instead.</p>
+
+ <p>If <c>Module:post_init_per_group/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_init_per_group(GroupName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -&gt; Result</name>
+ <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -&gt; Result</name>
<fsummary>Called before init_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
<v>InitData = Config | SkipOrFail</v>
<v>Config = NewConfig = [{Key,Value}]</v>
@@ -286,6 +299,11 @@
<seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>
instead.</p>
+ <p>If <c>Module:pre_init_per_testcase/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_init_per_testcase(TestcaseName,
+ InitData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
+
<p>CTHs cannot be added here right now. That feature may be added in
a later release, but it would right now break backwards
compatibility.</p>
@@ -293,9 +311,10 @@
</func>
<func>
- <name>Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after init_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -316,15 +335,21 @@
but for function
<seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>
instead.</p>
+
+ <p>If <c>Module:post_init_per_testcase/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_init_per_testcase(TestcaseName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -&gt; Result</name>
+ <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -&gt; Result</name>
<fsummary>Called before end_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
- <v>InitData = Config</v>
+ <v>EndData = Config</v>
<v>Config = NewConfig = [{Key,Value}]</v>
<v>CTHState = NewCTHState = term()</v>
<v>Result = {NewConfig, NewCTHState}</v>
@@ -345,14 +370,20 @@
<p>This function can not change the result of the test case by returning skip or fail
tuples, but it may insert items in <c>Config</c> that can be read in
- <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/4</c>.</p>
+ <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p>
+
+ <p>If <c>Module:pre_end_per_testcase/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_end_per_testcase(TestcaseName,
+ EndData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after end_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -373,13 +404,19 @@
but for function
<seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso>
instead.</p>
+
+ <p>If <c>Module:post_end_per_testcase/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_end_per_testcase(TestcaseName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:pre_end_per_group(GroupName, EndData, CTHState) -&gt; Result</name>
+ <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -&gt; Result</name>
<fsummary>Called before end_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>EndData = Config | SkipOrFail</v>
<v>Config = NewConfig = [{Key,Value}]</v>
@@ -400,13 +437,19 @@
but for function
<seealso marker="common_test#Module:end_per_group-2"><c>end_per_group</c></seealso>
instead.</p>
+
+ <p>If <c>Module:pre_end_per_group/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_end_per_group(GroupName,
+ EndData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after end_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -427,6 +470,11 @@
but for function
<seealso marker="common_test#Module:end_per_group-2">end_per_group</seealso>
instead.</p>
+
+ <p>If <c>Module:post_end_per_group/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_end_per_group(GroupName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
@@ -485,9 +533,10 @@
</func>
<func>
- <name>Module:on_tc_fail(TestName, Reason, CTHState) -&gt; NewCTHState</name>
+ <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -&gt; NewCTHState</name>
<fsummary>Called after the CTH scope ends.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>
<v>FuncName = atom()</v>
<v>GroupName = atom()</v>
@@ -505,7 +554,7 @@
<item><p>If <c>init_per_suite</c> fails, this function is called after
<seealso marker="#Module:post_init_per_suite-4"><c>post_init_per_suite</c></seealso>.</p></item>
<item><p>If a test case fails, this funcion is called after
- <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item>
+ <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>
</list>
<p>If the failed test case belongs to a test case group, the first
@@ -519,13 +568,19 @@
For details, see section
<seealso marker="event_handler_chapter#events">Event Handling</seealso>
in the User's Guide.</p>
+
+ <p>If <c>Module:on_tc_fail/4</c> is not exported, common_test
+ will attempt to call <c>Module:on_tc_fail(TestName, Reason,
+ CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:on_tc_skip(TestName, Reason, CTHState) -&gt; NewCTHState</name>
+ <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -&gt; NewCTHState</name>
<fsummary>Called after the CTH scope ends.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>
<v>FuncName = atom()</v>
<v>GroupName = atom()</v>
@@ -542,9 +597,9 @@
<list type="bulleted">
<item><p>If <c>init_per_group</c> is skipped, this function is
called after
- <seealso marker="#Module:post_init_per_group-4"><c>post_init_per_group</c></seealso>.</p></item>
+ <seealso marker="#Module:post_init_per_group-5"><c>post_init_per_group</c></seealso>.</p></item>
<item><p>If a test case is skipped, this function is called after
- <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item>
+ <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>
</list>
<p>If the skipped test case belongs to a test case group, the first
@@ -559,6 +614,11 @@
For details, see section
<seealso marker="event_handler_chapter#events">Event Handling</seealso>
in the User's Guide.</p>
+
+ <p>If <c>Module:on_tc_skip/4</c> is not exported, common_test
+ will attempt to call <c>Module:on_tc_skip(TestName, Reason,
+ CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml
index 0e4c35e11f..bfad96e489 100644
--- a/lib/common_test/doc/src/ct_hooks_chapter.xml
+++ b/lib/common_test/doc/src/ct_hooks_chapter.xml
@@ -38,7 +38,7 @@
extensions of the default behavior of <c>Common Test</c> using hooks
before and after all test suite calls. CTHs allow advanced <c>Common Test</c>
users to abstract out behavior that is common to multiple test suites
- without littering all test suites with library calls. this can be used
+ without littering all test suites with library calls. This can be used
for logging, starting, and monitoring external systems,
building C files needed by the tests, and so on.</p>
@@ -175,10 +175,10 @@
<row>
<cell><seealso marker="common_test#Module:init_per_group-2">
init_per_group/2</seealso></cell>
- <cell><seealso marker="ct_hooks#Module:post_init_per_group-4">
- post_init_per_group/4</seealso> is called</cell>
- <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4">
- post_end_per_group/4</seealso> has been called for that group</cell>
+ <cell><seealso marker="ct_hooks#Module:post_init_per_group-5">
+ post_init_per_group/5</seealso> is called</cell>
+ <cell><seealso marker="ct_hooks#Module:post_end_per_group-5">
+ post_end_per_group/5</seealso> has been called for that group</cell>
</row>
<tcaption>Scope of a CTH</tcaption>
</table>
@@ -245,16 +245,18 @@
</list>
<p>
- This is done in the CTH functions called pre_&lt;name of function&gt;.
- These functions take the same three arguments, <c>Name</c>,
+ This is done in the CTH functions called <c>pre_&lt;name of function&gt;</c>.
+ These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),
<c>Config</c>, and <c>CTHState</c>. The return value of the CTH function
is always a combination of a result for the suite/group/test and an
updated <c>CTHState</c>.</p>
<p>To let the test suite continue on executing, return the configuration
- list that you want the test to use as the result. To skip or
- fail the test, return a tuple with <c>skip</c> or <c>fail</c>, and a reason
- as the result.</p>
+ list that you want the test to use as the result.</p>
+
+ <p>All pre hooks, except <c>pre_end_per_testcase/4</c>, can
+ skip or fail the test by returning a tuple with <c>skip</c> or
+ <c>fail</c>, and a reason as the result.</p>
<p><em>Example:</em></p>
<code>
@@ -290,7 +292,7 @@
<p>
This is done in the CTH functions called <c>post_&lt;name of function&gt;</c>.
- These functions take the same four arguments, <c>Name</c>,
+ These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),
<c>Config</c>, <c>Return</c>, and <c>CTHState</c>. <c>Config</c> in this
case is the same <c>Config</c> as the testcase is called with.
<c>Return</c> is the value returned by the testcase. If the testcase
@@ -308,7 +310,7 @@
<p><em>Example:</em></p>
<code>
- post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -&gt;
+ post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) -&gt;
case db:check_consistency() of
true ->
%% DB is good, pass the test.
@@ -317,7 +319,7 @@
%% DB is not good, mark as skipped instead of failing
{{skip, "DB is inconsisten!"}, CTHState}
end;
- post_end_per_testcase(_TC, Config, Return, CTHState) -&gt;
+ post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) -&gt;
%% Do nothing if tc does not crash.
{Return, CTHState}.</code>
@@ -331,8 +333,8 @@
<title>Skip and Fail Hooks</title>
<p>
After any post hook has been executed for all installed CTHs,
- <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso>
- or <seealso marker="ct_hooks#Module:on_tc_skip-3">on_tc_skip</seealso>
+ <seealso marker="ct_hooks#Module:on_tc_fail-4">on_tc_fail</seealso>
+ or <seealso marker="ct_hooks#Module:on_tc_skip-4">on_tc_skip</seealso>
is called if the testcase failed or was skipped, respectively.
You cannot affect the outcome of the tests any further at this point.
</p>
@@ -389,18 +391,18 @@
-export([pre_end_per_suite/3]).
-export([post_end_per_suite/4]).
- -export([pre_init_per_group/3]).
- -export([post_init_per_group/4]).
- -export([pre_end_per_group/3]).
- -export([post_end_per_group/4]).
+ -export([pre_init_per_group/4]).
+ -export([post_init_per_group/5]).
+ -export([pre_end_per_group/4]).
+ -export([post_end_per_group/5]).
- -export([pre_init_per_testcase/3]).
- -export([post_init_per_testcase/4]).
- -export([pre_end_per_testcase/3]).
- -export([post_end_per_testcase/4]).
+ -export([pre_init_per_testcase/4]).
+ -export([post_init_per_testcase/5]).
+ -export([pre_end_per_testcase/4]).
+ -export([post_end_per_testcase/5]).
- -export([on_tc_fail/3]).
- -export([on_tc_skip/3]).
+ -export([on_tc_fail/4]).
+ -export([on_tc_skip/4]).
-export([terminate/1]).
@@ -435,46 +437,46 @@
total = State#state.total + State#state.suite_total } }.
%% @doc Called before each init_per_group.
- pre_init_per_group(Group,Config,State) ->
+ pre_init_per_group(Suite,Group,Config,State) ->
{Config, State}.
%% @doc Called after each init_per_group.
- post_init_per_group(Group,Config,Return,State) ->
+ post_init_per_group(Suite,Group,Config,Return,State) ->
{Return, State}.
%% @doc Called before each end_per_group.
- pre_end_per_group(Group,Config,State) ->
+ pre_end_per_group(Suite,Group,Config,State) ->
{Config, State}.
%% @doc Called after each end_per_group.
- post_end_per_group(Group,Config,Return,State) ->
+ post_end_per_group(Suite,Group,Config,Return,State) ->
{Return, State}.
%% @doc Called before each init_per_testcase.
- pre_init_per_testcase(TC,Config,State) ->
+ pre_init_per_testcase(Suite,TC,Config,State) ->
{Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.
%% Called after each init_per_testcase (immediately before the test case).
- post_init_per_testcase(TC,Config,Return,State) ->
+ post_init_per_testcase(Suite,TC,Config,Return,State) ->
{Return, State}
%% @doc Called before each end_per_testcase (immediately after the test case).
- pre_end_per_testcase(TC,Config,State) ->
+ pre_end_per_testcase(Suite,TC,Config,State) ->
{Config, State}.
%% @doc Called after each end_per_testcase.
- post_end_per_testcase(TC,Config,Return,State) ->
- TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)},
+ post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)},
{Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.
%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
- on_tc_fail(TC, Reason, State) ->
+ on_tc_fail(Suite, TC, Reason, State) ->
State.
%% @doc Called when a test case is skipped by either user action
%% or due to an init function failing.
- on_tc_skip(TC, Reason, State) ->
+ on_tc_skip(Suite, TC, Reason, State) ->
State.
%% @doc Called when the scope of the CTH is done