From a95a22017523229bb8924afbd928d3e16b564fa5 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Wed, 25 Jan 2017 11:24:58 +0100 Subject: [ct] Add Suite argument to hook callback functions An extra argument, Suite, is added as the first argument to each of the following hook callback functions: - pre_init_per_group - post_init_per_group - pre_end_per_group - post_end_per_group - pre_init_per_testcase - post_init_per_testcase - pre_end_per_testcase - post_end_per_testcase - on_tc_fail - on_tc_skip For backwards compatibility, if the new function is not exported from a hook callback module, common_test will fall back to the old interface and call the function without the Suite argument. The reason for adding the new argument is that if a test suite is skipped by a 'skip_suites' statement in the test specification, then there will be no call to pre/post_init_per_suite, and thus the hook has no other way of knowing which Suite is skipped when it gets the on_tc_skip callback. The other callbacks are updated for symmetry. --- lib/common_test/doc/src/ct_hooks.xml | 90 +++++++++++++++++++++++----- lib/common_test/doc/src/ct_hooks_chapter.xml | 74 ++++++++++++----------- 2 files changed, 113 insertions(+), 51 deletions(-) (limited to 'lib/common_test/doc/src') diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index c2cf29c530..a085f30262 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -208,9 +208,10 @@ - Module:pre_init_per_group(GroupName, InitData, CTHState) -> Result + Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -> Result Called before init_per_group. + SuiteName = atom() GroupName = atom() InitData = Config | SkipOrFail Config = NewConfig = [{Key,Value}] @@ -231,13 +232,19 @@ but for function init_per_group instead.

+ +

If Module:pre_init_per_group/4 is not exported, common_test + will attempt to call Module:pre_init_per_group(GroupName, + InitData, CTHState) instead. This is for backwards + compatibility.

 - Module:post_init_per_group(GroupName, Config, Return, CTHState) -> Result + Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result Called after init_per_group. + SuiteName = atom() GroupName = atom() Config = [{Key,Value}] Return = NewReturn = Config | SkipOrFail | term() @@ -258,13 +265,19 @@ but for function init_per_group instead.

+ +

If Module:post_init_per_group/5 is not exported, common_test + will attempt to call Module:post_init_per_group(GroupName, + Config, Return, CTHState) instead. This is for backwards + compatibility.

 - Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -> Result + Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -> Result Called before init_per_testcase. + SuiteName = atom() TestcaseName = atom() InitData = Config | SkipOrFail Config = NewConfig = [{Key,Value}] @@ -286,6 +299,11 @@ init_per_testcase instead.

+

If Module:pre_init_per_testcase/4 is not exported, common_test + will attempt to call Module:pre_init_per_testcase(TestcaseName, + InitData, CTHState) instead. This is for backwards + compatibility.

+

CTHs cannot be added here right now. That feature may be added in a later release, but it would right now break backwards compatibility.

@@ -293,9 +311,10 @@ - Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -> Result + Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result Called after init_per_testcase. + SuiteName = atom() TestcaseName = atom() Config = [{Key,Value}] Return = NewReturn = Config | SkipOrFail | term() @@ -316,15 +335,21 @@ but for function init_per_testcase instead.

+ +

If Module:post_init_per_testcase/5 is not exported, common_test + will attempt to call Module:post_init_per_testcase(TestcaseName, + Config, Return, CTHState) instead. This is for backwards + compatibility.

 - Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -> Result + Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -> Result Called before end_per_testcase. + SuiteName = atom() TestcaseName = atom() - InitData = Config + EndData = Config Config = NewConfig = [{Key,Value}] CTHState = NewCTHState = term() Result = {NewConfig, NewCTHState} @@ -345,14 +370,20 @@

This function can not change the result of the test case by returning skip or fail tuples, but it may insert items in Config that can be read in - end_per_testcase/2 or in post_end_per_testcase/4.

+ end_per_testcase/2 or in post_end_per_testcase/5.

+ +

If Module:pre_end_per_testcase/4 is not exported, common_test + will attempt to call Module:pre_end_per_testcase(TestcaseName, + EndData, CTHState) instead. This is for backwards + compatibility.

 - Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -> Result + Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result Called after end_per_testcase. + SuiteName = atom() TestcaseName = atom() Config = [{Key,Value}] Return = NewReturn = Config | SkipOrFail | term() @@ -373,13 +404,19 @@ but for function end_per_testcase instead.

+ +

If Module:post_end_per_testcase/5 is not exported, common_test + will attempt to call Module:post_end_per_testcase(TestcaseName, + Config, Return, CTHState) instead. This is for backwards + compatibility.

 - Module:pre_end_per_group(GroupName, EndData, CTHState) -> Result + Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -> Result Called before end_per_group. + SuiteName = atom() GroupName = atom() EndData = Config | SkipOrFail Config = NewConfig = [{Key,Value}] @@ -400,13 +437,19 @@ but for function end_per_group instead.

+ +

If Module:pre_end_per_group/4 is not exported, common_test + will attempt to call Module:pre_end_per_group(GroupName, + EndData, CTHState) instead. This is for backwards + compatibility.

 - Module:post_end_per_group(GroupName, Config, Return, CTHState) -> Result + Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result Called after end_per_group. + SuiteName = atom() GroupName = atom() Config = [{Key,Value}] Return = NewReturn = Config | SkipOrFail | term() @@ -427,6 +470,11 @@ but for function end_per_group instead.

+ +

If Module:post_end_per_group/5 is not exported, common_test + will attempt to call Module:post_end_per_group(GroupName, + Config, Return, CTHState) instead. This is for backwards + compatibility.

 @@ -485,9 +533,10 @@ - Module:on_tc_fail(TestName, Reason, CTHState) -> NewCTHState + Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -> NewCTHState Called after the CTH scope ends. + SuiteName = atom() TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName FuncName = atom() GroupName = atom() @@ -505,7 +554,7 @@

If init_per_suite fails, this function is called after post_init_per_suite.

If a test case fails, this funcion is called after - post_end_per_testcase.

 + post_end_per_testcase.

If the failed test case belongs to a test case group, the first @@ -519,13 +568,19 @@ For details, see section Event Handling in the User's Guide.

+ +

If Module:on_tc_fail/4 is not exported, common_test + will attempt to call Module:on_tc_fail(TestName, Reason, + CTHState) instead. This is for backwards + compatibility.

 - Module:on_tc_skip(TestName, Reason, CTHState) -> NewCTHState + Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -> NewCTHState Called after the CTH scope ends. + SuiteName = atom() TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName FuncName = atom() GroupName = atom() @@ -542,9 +597,9 @@

If init_per_group is skipped, this function is called after - post_init_per_group.

 + post_init_per_group.

If a test case is skipped, this function is called after - post_end_per_testcase.

 + post_end_per_testcase.

If the skipped test case belongs to a test case group, the first @@ -559,6 +614,11 @@ For details, see section Event Handling in the User's Guide.

+ +

If Module:on_tc_skip/4 is not exported, common_test + will attempt to call Module:on_tc_skip(TestName, Reason, + CTHState) instead. This is for backwards + compatibility.

 diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index 0e4c35e11f..bfad96e489 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -38,7 +38,7 @@ extensions of the default behavior of Common Test using hooks before and after all test suite calls. CTHs allow advanced Common Test users to abstract out behavior that is common to multiple test suites - without littering all test suites with library calls. this can be used + without littering all test suites with library calls. This can be used for logging, starting, and monitoring external systems, building C files needed by the tests, and so on.

@@ -175,10 +175,10 @@ init_per_group/2 - - post_init_per_group/4 is called - - post_end_per_group/4 has been called for that group + + post_init_per_group/5 is called + + post_end_per_group/5 has been called for that group Scope of a CTH @@ -245,16 +245,18 @@

- This is done in the CTH functions called pre_<name of function>. - These functions take the same three arguments, Name, + This is done in the CTH functions called pre_<name of function>. + These functions take the arguments SuiteName, Name (group or test case name, if applicable), Config, and CTHState. The return value of the CTH function is always a combination of a result for the suite/group/test and an updated CTHState.

To let the test suite continue on executing, return the configuration - list that you want the test to use as the result. To skip or - fail the test, return a tuple with skip or fail, and a reason - as the result.

+ list that you want the test to use as the result.

+ +

All pre hooks, except pre_end_per_testcase/4, can + skip or fail the test by returning a tuple with skip or + fail, and a reason as the result.

Example:

@@ -290,7 +292,7 @@

This is done in the CTH functions called post_<name of function>. - These functions take the same four arguments, Name, + These functions take the arguments SuiteName, Name (group or test case name, if applicable), Config, Return, and CTHState. Config in this case is the same Config as the testcase is called with. Return is the value returned by the testcase. If the testcase @@ -308,7 +310,7 @@

Example:

- post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) -> case db:check_consistency() of true -> %% DB is good, pass the test. @@ -317,7 +319,7 @@ %% DB is not good, mark as skipped instead of failing {{skip, "DB is inconsisten!"}, CTHState} end; - post_end_per_testcase(_TC, Config, Return, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) -> %% Do nothing if tc does not crash. {Return, CTHState}. @@ -331,8 +333,8 @@ Skip and Fail Hooks

After any post hook has been executed for all installed CTHs, - on_tc_fail - or on_tc_skip + on_tc_fail + or on_tc_skip is called if the testcase failed or was skipped, respectively. You cannot affect the outcome of the tests any further at this point.

@@ -389,18 +391,18 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). - -export([pre_init_per_group/3]). - -export([post_init_per_group/4]). - -export([pre_end_per_group/3]). - -export([post_end_per_group/4]). + -export([pre_init_per_group/4]). + -export([post_init_per_group/5]). + -export([pre_end_per_group/4]). + -export([post_end_per_group/5]). - -export([pre_init_per_testcase/3]). - -export([post_init_per_testcase/4]). - -export([pre_end_per_testcase/3]). - -export([post_end_per_testcase/4]). + -export([pre_init_per_testcase/4]). + -export([post_init_per_testcase/5]). + -export([pre_end_per_testcase/4]). + -export([post_end_per_testcase/5]). - -export([on_tc_fail/3]). - -export([on_tc_skip/3]). + -export([on_tc_fail/4]). + -export([on_tc_skip/4]). -export([terminate/1]). @@ -435,46 +437,46 @@ total = State#state.total + State#state.suite_total } }. %% @doc Called before each init_per_group. - pre_init_per_group(Group,Config,State) -> + pre_init_per_group(Suite,Group,Config,State) -> {Config, State}. %% @doc Called after each init_per_group. - post_init_per_group(Group,Config,Return,State) -> + post_init_per_group(Suite,Group,Config,Return,State) -> {Return, State}. %% @doc Called before each end_per_group. - pre_end_per_group(Group,Config,State) -> + pre_end_per_group(Suite,Group,Config,State) -> {Config, State}. %% @doc Called after each end_per_group. - post_end_per_group(Group,Config,Return,State) -> + post_end_per_group(Suite,Group,Config,Return,State) -> {Return, State}. %% @doc Called before each init_per_testcase. - pre_init_per_testcase(TC,Config,State) -> + pre_init_per_testcase(Suite,TC,Config,State) -> {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. %% Called after each init_per_testcase (immediately before the test case). - post_init_per_testcase(TC,Config,Return,State) -> + post_init_per_testcase(Suite,TC,Config,Return,State) -> {Return, State} %% @doc Called before each end_per_testcase (immediately after the test case). - pre_end_per_testcase(TC,Config,State) -> + pre_end_per_testcase(Suite,TC,Config,State) -> {Config, State}. %% @doc Called after each end_per_testcase. - post_end_per_testcase(TC,Config,Return,State) -> - TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + post_end_per_testcase(Suite,TC,Config,Return,State) -> + TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)}, {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. - on_tc_fail(TC, Reason, State) -> + on_tc_fail(Suite, TC, Reason, State) -> State. %% @doc Called when a test case is skipped by either user action %% or due to an init function failing. - on_tc_skip(TC, Reason, State) -> + on_tc_skip(Suite, TC, Reason, State) -> State. %% @doc Called when the scope of the CTH is done -- cgit v1.2.3