diff options
| -rw-r--r-- | lib/common_test/doc/src/Makefile | 1 | ||||
| -rw-r--r-- | lib/common_test/doc/src/common_test_app.xml | 10 | ||||
| -rw-r--r-- | lib/common_test/doc/src/ct_suite_callbacks.xml | 503 | ||||
| -rw-r--r-- | lib/common_test/doc/src/event_handler_chapter.xml | 11 | ||||
| -rw-r--r-- | lib/common_test/doc/src/part.xml | 1 | ||||
| -rw-r--r-- | lib/common_test/doc/src/run_test_chapter.xml | 11 | ||||
| -rw-r--r-- | lib/common_test/doc/src/suite_callbacks_chapter.xml | 394 | ||||
| -rw-r--r-- | lib/common_test/doc/src/write_test_chapter.xml | 1 | ||||
| -rw-r--r-- | lib/common_test/src/ct.erl | 6 |
9 files changed, 910 insertions, 28 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index be066143e0..a3f740852c 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -71,6 +71,7 @@ XML_CHAPTER_FILES = \ cover_chapter.xml \ ct_master_chapter.xml \ event_handler_chapter.xml \ + suite_callbacks_chapter.xml \ dependencies_chapter.xml \ notes.xml \ notes_history.xml diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index e30eef2488..a735dc85a8 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -131,7 +131,8 @@ <type> <v> Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | - {silent_connections,Conns} | {stylesheet,CSSFile}</v> + {silent_connections,Conns} | {stylesheet,CSSFile} | + {suite_callbacks, SCBs}</v> <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()}</v> <v> MilliSec = integer()</v> @@ -143,6 +144,9 @@ <v> UserData = term()</v> <v> Conns = [atom()]</v> <v> CSSFile = string()</v> + <v> SCBs = [SCBModule | {SCBModule, SCBInitArgs}]</v> + <v> SCBModule = atom()</v> + <v> SCBInitArgs = term()</v> </type> <desc> @@ -170,6 +174,10 @@ <p>With <c>userdata</c>, it is possible for the user to specify arbitrary test suite related information which can be read by calling <c>ct:userdata/2</c>.</p> + + <p>The <c>suite_callbacks</c> tag specifies which + <seealso marker="suite_callbacks_chapter">Suite Callbacks</seealso> + are to be run together with this suite.</p> <p>Other tuples than the ones defined will simply be ignored.</p> diff --git a/lib/common_test/doc/src/ct_suite_callbacks.xml b/lib/common_test/doc/src/ct_suite_callbacks.xml index 28a496a47b..cea5804825 100644 --- a/lib/common_test/doc/src/ct_suite_callbacks.xml +++ b/lib/common_test/doc/src/ct_suite_callbacks.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> @@ -33,24 +33,43 @@ <file>suite_callback.sgml</file> </header> <module>ct_suite_callback</module> - <modulesummary>A callback interface on top of common test</modulesummary> + <modulesummary>A callback interface on top of Common Test</modulesummary> <description> - <p>The <em>Suite Callback</em> framework allows extensions of the default - behaviour of Common Test by means of callbacks before and after all - test suite calls. </p> + <warning><p>This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.</p></warning> + + <p>The <em>Suite Callback</em> (henceforth called SCB) framework allows + extensions of the default behaviour of Common Test by means of callbacks + before and after all test suite calls. It is meant for advanced users of + Common Test which want to abstract out behaviour which is common to + multiple test suites. </p> <p>In brief, Suite Callbacks allows you to:</p> <list> - <item>Manipulating the runtime config before each suite configuration calls</item> - <item>Manipulating the return of all suite configuration calls</item> + <item>Manipulate the runtime config before each suite + configuration calls</item> + <item>Manipulate the return of all suite configuration calls and in + extension the result of the test themselves.</item> </list> - <p>The following sections describe the mandatory and optional suite callback - functions Common Test will call during test execution. For more details - see <seealso marker="suite_callbacks">Suite Callbacks</seealso> in the User's Guide.</p> + <p>The following sections describe the mandatory and optional SCB + functions Common Test will call during test execution. For more details + see <seealso marker="suite_callbacks_chapter">Suite Callbacks</seealso> in + the User's Guide.</p> + + <p>For information about how to add a SCB to your suite see + <seealso marker="suite_callbacks_chapter#installing">Installing an SCB + </seealso> in the User's Guide.</p> + + <note><p>See the + <seealso marker="suite_callbacks_chapter#example">Example SCB</seealso> + in the User's Guide for a minimal example of an SCB. </p></note> </description> @@ -62,31 +81,469 @@ <funcs> <func> - <name>Module:init(Opts) -> {Id,State} </name> - <fsummary>asdas</fsummary> + <name>Module:init(Id, Opts) -> State</name> + <fsummary>Initiates the Suite Callback</fsummary> <type> + <v>Id = reference() | term()</v> <v>Opts = term()</v> - <v>Id = term()</v> - <v>GroupName = term()</v> + <v>State = term()</v> </type> <desc> <p> MANDATORY </p> + + <p>Always called before any other callback function. + Use this to initiate any common state. + It should return a state for this SCB.</p> + + <p><c>Id</c> is the return value of + <seealso marker="#Module:id-1">id/1</seealso>, or a <c>reference</c> + if <seealso marker="#Module:id-1">id/1</seealso> is not implemented. + </p> + + <p>For details about when init is called see + <seealso marker="suite_callbacks_chapter#scope">scope</seealso> + in the User's Guide.</p> + + </desc> + </func> + + <func> + <name>Module:pre_init_per_suite(SuiteName, Config, SCBState) -> + Result</name> + <fsummary>Called before init_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {Return, NewSCBState}</v> + <v>Return = NewConfig | SkipOrFail</v> + <v>SkipOrFail = {fail, Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> if it exists. + It typically contains initialization/logging which needs to be done + before init_per_suite is called. + If <c>{skip,Reason}</c> or <c>{fail,Reason}</c> is returned, + init_per_suite and all test cases of the suite will be skipped and + Reason printed in the overview log of the suite.</p> - <p></p> + <p><c>SuiteName</c> is the name of the suite to be run.</p> + + <p><c>Config</c> is the original config list of the test suite.</p> + + <p><c>SCBState</c> is the current internal state of the SCB.</p> + + <p><c>Return</c> is the result of the init_per_suite function. + If it is <c>{skip,Reason}</c> or <c>{fail,Reason}</c> + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> will never be called, instead the initiation is considered + to be skipped/failed respectively. If a <c>NewConfig</c> list + is returned, <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> will be called with that <c>NewConfig</c> list. + See <seealso marker="suite_callbacks_chapter#manipulating"> + Manipulating tests</seealso> in the User's Guide for more details.</p> + + + <p>Note that this function is only called if the SCB has been added + before init_per_suite is run, see + <seealso marker="suite_callbacks_chapter#scope">SCB Scoping</seealso> + in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:post_init_per_suite(SuiteName, Config, Return, SCBState) -> + Result</name> + <fsummary>Called after init_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail, Reason} | {skip, Reason} | term()</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewReturn, NewSCBState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p><c>Return</c> is what + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> returned, i.e. {fail,Reason}, {skip,Reason}, a <c>Config</c> + list or a term describing how + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> failed.</p> + + <p><c>NewReturn</c> is the possibly modified return value of + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso>. It is here possible to recover from a failure in + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> by returning the <c>ConfigList</c> with the <c>tc_status</c> + element removed.</p> + + <p><c>SCBState</c> is the current internal state of the SCB.</p> + + <p>This function is called after + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> if it exists.</p> + + <p>Note that this function is only called if the SCB has been added + before or in init_per_suite, see + <seealso marker="suite_callbacks_chapter#scope">SCB Scoping</seealso> + in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:pre_init_per_group(GroupName, Config, SCBState) -> + Result</name> + <fsummary>Called before init_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> - <p> If <c>{skip,Reason}</c> is returned, all test cases - in the module will be skipped, and the <c>Reason</c> will - be printed on the HTML result page.</p> + <p>This function is called before + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> instead.</p> + </desc> + </func> + + <func> + <name>Module:post_init_per_group(GroupName, Config, Return, SCBState) -> + Result</name> + <fsummary>Called after init_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewReturn, NewSCBState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> - <p>For details on groups, see - <seealso marker="write_test_chapter#test_case_groups">Test case - groups</seealso> in the User's Guide.</p> + <p>This function is called after + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_init_per_testcase(TestcaseName, Config, SCBState) -> + Result</name> + <fsummary>Called before init_per_testcase</fsummary> + <type> + <v>TestcaseName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> - </desc> + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_testcase-2"> + init_per_testcase</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_testcase-2"> + init_per_testcase</seealso> function instead.</p> + + <p>Note that it is not possible to add SCB's here right now, + that feature might be added later, + but it would right now break backwards compatability.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_testcase(TestcaseName, Config, Return, SCBState) + -> Result</name> + <fsummary>Called after end_per_testcase</fsummary> + <type> + <v>TestcaseName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewReturn, NewSCBState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_testcase-2"> + end_per_testcase</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_testcase-2"> + end_per_testcase</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_end_per_group(GroupName, Config, SCBState) -> + Result</name> + <fsummary>Called before end_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_group(GroupName, Config, Return, SCBState) -> + Result</name> + <fsummary>Called after end_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewReturn, NewSCBState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_end_per_suite(SuiteName, Config, SCBState) -> + Result</name> + <fsummary>Called before end_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewSCBState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_suite-2"> + end_per_suite</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_suite(SuiteName, Config, Return, SCBState) -> + Result</name> + <fsummary>Called after end_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>SCBState = NewSCBState = term()</v> + <v>Result = {NewReturn, NewSCBState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> if it exists. It behaves the same way as + <seealso marker="ct_suite_callback#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_suite-2"> + end_per_suite</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:on_tc_fail(TestcaseName, Reason, SCBState) -> + NewSCBState</name> + <fsummary>Called after the SCB scope ends</fsummary> + <type> + <v>TestcaseName = init_per_suite | end_per_suite | + init_per_group | end_per_group | atom()</v> + <v>Reason = term()</v> + <v>SCBState = NewSCBState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called whenever a testcase fails. + It is called after the post function has been called for + the testcase which failed. i.e. + if init_per_suite fails this function is called after + <seealso marker="#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, and if a testcase fails it is called + after <seealso marker="#Module:post_end_per_testcase-4"> + post_end_per_testcase</seealso>.</p> + + <p>The data which comes with the Reason follows the same format as the + <seealso marker="event_handler_chapter#failreason">FailReason + </seealso> in the <seealso marker="event_handler_chapter#tc_done">tc_done</seealso> event. + See <seealso marker="event_handler_chapter#events">Event Handling + </seealso> in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:on_tc_skip(TestcaseName, Reason, SCBState) -> + NewSCBState</name> + <fsummary>Called after the SCB scope ends</fsummary> + <type> + <v>TestcaseName = end_per_suite | init_per_group | + end_per_group | atom()</v> + <v>Reason = {tc_auto_skip | tc_user_skip, term()}</v> + <v>SCBState = NewSCBState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called whenever a testcase is skipped. + It is called after the post function has been called for the + testcase which was skipped. + i.e. if init_per_group is skipped this function is called after + <seealso marker="#Module:post_init_per_suite-4">post_init_per_group + </seealso>, and if a testcase is skipped it is called after + <seealso marker="#Module:post_end_per_testcase-4">post_end_per_testcase + </seealso>.</p> + + <p>The data which comes with the Reason follows the same format as + <seealso marker="event_handler_chapter#tc_auto_skip">tc_auto_skip + </seealso> and <seealso marker="event_handler_chapter#tc_user_skip"> + tc_user_skip</seealso> events. + See <seealso marker="event_handler_chapter#events">Event Handling + </seealso> in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:terminate(SCBState)</name> + <fsummary>Called after the SCB scope ends</fsummary> + <type> + <v>SCBState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called at the end of an SCB's + <seealso marker="suite_callbacks_chapter#scope">scope</seealso>. + </p> + </desc> + </func> + + <func> + <name>Module:id(Opts) -> Id</name> + <fsummary>Called before the init function of an SCB</fsummary> + <type> + <v>Opts = term()</v> + <v>Id = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>The <c>Id</c> is used to uniquely identify an SCB instance, + if two SCB's return the same <c>Id</c> the second SCB is ignored + and subsequent calls to the SCB will only be made to the first + instance. For more information see + <seealso marker="suite_callbacks_chapter#installing">Installing an SCB + </seealso> in the User's Guide. + </p> + + <p>This function should NOT have any side effects as it might + be called multiple times by Common Test.</p> + + <p>If not implemented the SCB will act as if this function returned a + call to <c>make_ref/0</c>.</p> + </desc> </func> - </funcs> </erlref> diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index 904876ac46..a01feb59d1 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -61,6 +61,7 @@ itself.</p> </section> <section> + <marker id="usage"></marker> <title>Usage</title> <p>Event handlers may be installed by means of an <c>event_handler</c> start flag (<c>ct_run</c>) or option (<c>ct:run_test/1</c>), where the @@ -120,6 +121,7 @@ node the event has originated from (only relevant for CT Master event handlers). <c>data</c> is specific for the particular event.</p> + <marker id="events"></marker> <p><em>General events:</em></p> <list> @@ -172,6 +174,7 @@ are also given. </p></item> + <marker id="tc_done"/> <item><c>#event{name = tc_done, data = {Suite,FuncOrGroup,Result}}</c> <p><c>Suite = atom()</c>, name of the suite.</p> <p><c>FuncOrGroup = Func | {Conf,GroupName,GroupProperties}</c></p> @@ -181,12 +184,14 @@ (unknown if init- or end function times out).</p> <p><c>GroupProperties = list()</c>, list of execution properties for the group.</p> <p><c>Result = ok | {skipped,SkipReason} | {failed,FailReason}</c>, the result.</p> + <marker id="skipreason"/> <p><c>SkipReason = {require_failed,RequireInfo} | {require_failed_in_suite0,RequireInfo} | {failed,{Suite,init_per_testcase,FailInfo}} | UserTerm</c>, the reason why the case has been skipped.</p> - <p><c>FailReason = {error,FailInfo} | + <marker id="failreason"/> + <p><c>FailReason = {error,FailInfo} | {error,{RunTimeError,StackTrace}} | {timetrap_timeout,integer()} | {failed,{Suite,end_per_testcase,FailInfo}}</c>, reason for failure.</p> @@ -209,6 +214,7 @@ <c>end_per_testcase</c> for the case failed. </p></item> + <marker id="tc_auto_skip"></marker> <item><c>#event{name = tc_auto_skip, data = {Suite,Func,Reason}}</c> <p><c>Suite = atom()</c>, the name of the suite.</p> <p><c>Func = atom()</c>, the name of the test case or configuration function.</p> @@ -234,7 +240,8 @@ skipped because of <c>init_per_testcase</c> failing, since that information is carried with the <c>tc_done</c> event. </p></item> - + + <marker id="tc_user_skip"></marker> <item><c>#event{name = tc_user_skip, data = {Suite,TestCase,Comment}}</c> <p><c>Suite = atom()</c>, name of the suite.</p> <p><c>TestCase = atom()</c>, name of the test case.</p> diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 53a4cb1bbf..1a09ec1da2 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -75,6 +75,7 @@ <xi:include href="ct_master_chapter.xml"/> <xi:include href="event_handler_chapter.xml"/> <xi:include href="dependencies_chapter.xml"/> + <xi:include href="suite_callbacks_chapter.xml"/> <xi:include href="why_test_chapter.xml"/> </part> diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 94fcf6bf01..81e752680b 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -105,6 +105,7 @@ RPC from a remote node.</p> </section> + <marker id="ct_run"></marker> <section> <title>Running tests from the OS command line</title> @@ -147,6 +148,8 @@ <seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item> <item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install <seealso marker="event_handler_chapter#event_handling">event handlers</seealso> including start arguments.</item> + <item><c><![CDATA[-suite_callbacks <suite_callbacks>]]></c>, to install + <seealso marker="suite_callbacks_chapter#installing">Suite Callbacks</seealso> including start arguments.</item> <item><c><![CDATA[-include]]></c>, specifies include directories (see above).</item> <item><c><![CDATA[-no_auto_compile]]></c>, disables the automatic test suite compilation feature (see above).</item> <item><c><![CDATA[-multiply_timetraps <n>]]></c>, extends <seealso marker="write_test_chapter#timetraps">timetrap @@ -333,8 +336,8 @@ with <c>dir</c>.</p> </section> + <marker id="test_specifications"></marker> <section> - <marker id="test_specifications"></marker> <title>Using test specifications</title> <p>The most flexible way to specify what to test, is to use a so @@ -440,6 +443,9 @@ {event_handler, NodeRefs, EventHandlers}. {event_handler, EventHandlers, InitArgs}. {event_handler, NodeRefs, EventHandlers, InitArgs}. + + {suite_callbacks, SCBModules}. + {suite_callbacks, NodeRefs, SCBModules}. </pre> <p>Test terms:</p> <pre> @@ -478,6 +484,9 @@ LogDir = string() EventHandlers = atom() | [atom()] InitArgs = [term()] + SCBModules = [SCBModule | {SCBModule, SCBInitArgs}] + SCBModule = atom() + SCBInitArgs = term() DirRef = DirAlias | Dir Suites = atom() | [atom()] | all Suite = atom() diff --git a/lib/common_test/doc/src/suite_callbacks_chapter.xml b/lib/common_test/doc/src/suite_callbacks_chapter.xml new file mode 100644 index 0000000000..89f78898d4 --- /dev/null +++ b/lib/common_test/doc/src/suite_callbacks_chapter.xml @@ -0,0 +1,394 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2011</year><year>2011</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Suite Callbacks</title> + <prepared>Lukas Larsson</prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>suite_callbacks_chapter.xml</file> + </header> + + <marker id="general"></marker> + <section> + <title>General</title> + <warning><p>This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.</p></warning> + <p> + The <em>Suite Callback</em> (henceforth called SCB) framework allows + extensions of the default behaviour of Common Test by means of callbacks + before and after all test suite calls. SCBs allow advanced Common Test + users to abstract out behaviour which is common to multiple test suites + without littering all test suites with library calls. Some example + usages are: logging, starting and monitoring external systems, + building C files needed by the tests and much more!</p> + + <p>In brief, Suite Callbacks allows you to:</p> + + <list> + <item>Manipulate the runtime config before each suite + configuration calls</item> + <item>Manipulate the return of all suite configuration calls and in + extension the result of the test themselves.</item> + </list> + + <p>The following sections describe how to use SCBs, when they are run + and how to manipulate your test results in an SCB</p> + + <warning><p>When executing within an SCB all timetraps are shutoff. So + if your SCB never returns, the entire test run will be stalled!</p> + </warning> + + </section> + + <marker id="installing"></marker> + <section> + <title>Installing an SCB</title> + <p>There are multiple ways to install an SCB in your test run. You can do it + for all tests in a run, for specific test suites and for specific groups + within a test suite. If you want an SCB to be present in all test suites + within your testrun there are three different ways to accomplish that.</p> + + <list> + <item>Add <c>-suite_callbacks</c> as an argument to + <seealso marker="run_test_chapter#ct_run">ct_run</seealso>. + To add multiple SCBs using this method append them to each other + using the keyword <c>and</c>, i.e. + <c>ct_run -suite_callbacks scb1 [{debug,true}] and scb2 ...</c>.</item> + <item>Add the <c>suite_callbacks</c> tag to your + <seealso marker="run_test_chapter#test_specifications"> + Test Specification</seealso></item> + <item>Add the <c>suite_callbacks</c> tag to your call to + <seealso marker="ct#run_test-1">ct:run_test/1</seealso></item> + </list> + + <p>You can also add SCBs within a test suite. This is done by returning + <c>{suite_callbacks,[SCB]}</c> in the config list from + <seealso marker="common_test#Module:suite-0">suite/0</seealso>, + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite/1</seealso> or + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group/2</seealso>. <c>SCB</c> in this case can be either + only the module name of the SCB or a tuple with the module name and the + initial arguments to the SCB. Eg: + <c>{suite_callbacks,[my_scb_module]}</c> or + <c>{suite_callbacks,[{my_scb_module,[{debug,true}]}]}</c></p> + + <section> + <title>Overriding SCBs</title> + <p>By default each installation of an SCB will cause a new instance of it + to be activated. This can cause problems if you want to be able to + override SCBs in testspecifications while still having them in the + suite info function. The + <seealso marker="ct_suite_callback#Module:id-1">id/1</seealso> + callback exists to address this problem. By returning the same + <c>id</c> in both places, Common Test knows that this SCB + has already been installed and will not try to install it again.</p> + </section> + + </section> + + <marker id="scope"/> + <section> + <title>SCB Scope</title> + <p>Once the SCB is installed into a certain test run it will be there until + it's scope is expired. The scope of an SCB depends on when it is + installed. + The <seealso marker="ct_suite_callback#Module:init-2">init/2</seealso> is + called at the beginning of the scope and the + <seealso marker="ct_suite_callback#Module:terminate-1">terminate/1 + </seealso> function is called when the scope ends.</p> + <table> + <row> + <cell><em>SCB Installed in</em></cell> + <cell><em>SCB scope begins before</em></cell> + <cell><em>SCB scope ends after</em></cell> + </row> + <row> + <cell><seealso marker="run_test_chapter#ct_run">ct_run</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="ct#run_test-1">ct:run_test</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="run_test_chapter#test_specifications"> + Test Specification</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:suite-0">suite/0 + </seealso></cell> + <cell><seealso marker="ct_suite_callback#Module:pre_init_per_suite-3"> + pre_init_per_suite/3</seealso> is called.</cell> + <cell><seealso marker="ct_suite_callback#Module:post_end_per_suite-4"> + post_end_per_suite/4</seealso> has been called for that test suite.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite/1</seealso></cell> + <cell><seealso marker="ct_suite_callback#Module:post_init_per_suite-4"> + post_init_per_suite/4</seealso> is called.</cell> + <cell><seealso marker="ct_suite_callback#Module:post_end_per_suite-4"> + post_end_per_suite/4</seealso> has been called for that test suite.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:init_per_group-2"> + init_per_group/2</seealso></cell> + <cell><seealso marker="ct_suite_callback#Module:post_init_per_group-4"> + post_init_per_group/4</seealso> is called.</cell> + <cell><seealso marker="ct_suite_callback#Module:post_end_per_suite-4"> + post_end_per_group/4</seealso> has been called for that group.</cell> + </row> + <tcaption>Scope of an SCB</tcaption> + </table> + + <section> + <title>CTH Processes and Tables</title> + <p>CTHs are run with the same process scoping as normal test suites + i.e. a different process will execute the init_per_suite hooks then the + init_per_group or per_testcase hooks. So if you want to spawn a + process in the CTH you cannot link with the CTH process as it will exit + after the post hook ends. Also if you for some reason need an ETS + table with your CTH, you will have to spawn a process which handles + it.</p> + </section> + + </section> + + <marker id="manipulating"/> + <section> + <title>Manipulating tests</title> + <p>It is through SCB's possible to manipulate the results of tests and + configuration functions. The main purpose of doing this with SCBs is to + allow common patterns to be abstracted out from test test suites and applied to + multiple test suites without duplicating any code. All of the callback + functions for an SCB follow a common interface, this interface is + described below.</p> + + <marker id="pre"/> + <section> + <title>Pre test manipulation</title> + <p> + It is possible in an SCB to hook in behaviour before + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_testcase</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and + <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>. + This is done in the SCB functions called pre_<name of function>. + All of these function take the same three arguments: <c>Name</c>, + <c>Config</c> and <c>SCBState</c>. The return value of the SCB function + is always a combination of an result for the suite/group/test and an + updated <c>SCBState</c>. If you want the test suite to continue on + executing you should return the config list which you want the test to + use as the result. If you for some reason want to skip/fail the test, + return a tuple with <c>skip</c> or <c>fail</c> and a reason as the + result. Example: + </p> + <code>pre_init_per_suite(SuiteName, Config, SCBState) -> + case db:connect() of + {error,_Reason} -> + {{fail, "Could not connect to DB"}, SCBState}; + {ok, Handle} -> + {[{db_handle, Handle} | Config], SCBState#state{ handle = Handle }} + end.</code> + + </section> + + <marker id="post"/> + <section> + <title>Post test manipulation</title> + <p>It is also possible in an SCB to hook in behaviour after + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_testcase</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and + <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>. + This is done in the SCB functions called post_<name of function>. + All of these function take the same four arguments: <c>Name</c>, + <c>Config</c>, <c>Return</c> and <c>SCBState</c>. <c>Config</c> in this + case is the same <c>Config</c> as the testcase is called with. + <c>Return</c> is the value returned by the testcase. If the testcase + failed by crashing, <c>Return</c> will be + <c>{'EXIT',{{Error,Reason},Stacktrace}}</c>.</p> + + <p>The return value of the SCB function is always a combination of an + result for the suite/group/test and an updated <c>SCBState</c>. If + you want the callback to not affect the outcome of the test you should + return the <c>Return</c> data as it is given to the SCB. You can also + modify the result of the test. By returning the <c>Config</c> list + with the <c>tc_status</c> element removed you can recover from a test + failure. As in all the pre hooks, it is also possible to fail/skip + the test case in the post hook. Example: </p> + + <code>post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, SCBState) -> + case db:check_consistency() of + true -> + %% DB is good, pass the test. + {proplists:delete(tc_status, Config), SCBState}; + false -> + %% DB is not good, mark as skipped instead of failing + {{skip, "DB is inconsisten!"}, SCBState} + end; +post_end_per_testcase(_TC, Config, Return, SCBState) -> + %% Do nothing if tc does not crash. + {Return, SCBState}.</code> + + <note>Recovering from a testcase failure using SCBs should only be done as + a last resort. If used wrongly it could become very difficult to + determine which tests pass or fail in a test run</note> + + </section> + + <marker id="skip_n_fail"/> + <section> + <title>Skip and Fail</title> + <p> + After any post hook has been executed for all installed SCBs, + <seealso marker="ct_suite_callback#Module:on_tc_fail-3">on_tc_fail</seealso> + or <seealso marker="ct_suite_callback#Module:on_tc_fail-3">on_tc_skip</seealso> + might be called if the testcase failed or was skipped + respectively. You cannot affect the outcome of the tests any further at + this point. + </p> + </section> + + </section> + + <marker id="example"/> + <section> + <title>Example SCB</title> + <p>The SCB below will log information about a test run into a format + parseable by <seealso marker="kernel:file#consult-1">file:consult/1</seealso>. + </p> + <code>%%% @doc Common Test Example Suite Callback module. +-module(example_scb). + +%% Suite Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3]). + +-export([terminate/1]). + +-record(state, { file_handle, total, suite_total, ts, tcs, data }). + +%% @doc Return a unique id for this SCB. +id(Opts) -> + proplists:get_value(filename, Opts, "/tmp/file.log"). + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, Opts) -> + {ok,D} = file:open(Id,[write]), + #state{ file_handle = D, total = 0, data = [] }. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{ suite_total = 0, tcs = [] }}. + +%% @doc Called after init_per_suite. +post_init_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(Suite,Config,Return,State) -> + Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)}, + {Return, State#state{ data = [Data | State#state.data] , + total = State#state.total + State#state.suite_total } }. + +%% @doc Called before each init_per_group. +pre_init_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called before each test case. +pre_init_per_testcase(TC,Config,State) -> + {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. + +%% @doc Called after each test case. +post_end_per_testcase(TC,Config,Return,State) -> + TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(TC, Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. +on_tc_skip(TC, Reason, State) -> + State. + +%% @doc Called when the scope of the SCB is done +terminate(State) -> + io:format(State#state.file_handle, "~p.~n", + [{test_run, State#state.total, State#state.data}]), + file:close(State#state.file_handle), + ok.</code> + </section> + +</chapter> + + + + diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 5afec6de6a..76493d3616 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -115,6 +115,7 @@ </p> </section> + <marker id="per_testcase"/> <section> <title>Init and end per test case</title> diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 405dc40c8b..eb0eceeb46 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -148,7 +148,8 @@ run(TestDirs) -> %%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} | %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | -%%% {refresh_logs,LogDir} | {basic_html,Bool} +%%% {refresh_logs,LogDir} | {basic_html,Bool} | +%%% {suite_callbacks, SCBs} %%% TestDirs = [string()] | string() %%% Suites = [string()] | string() %%% Cases = [atom()] | atom() @@ -176,6 +177,9 @@ run(TestDirs) -> %%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile} %%% DecryptKey = string() %%% DecryptFile = string() +%%% SCBs = [SCBModule | {SCBModule, SCBInitArgs}] +%%% SCBModule = atom() +%%% SCBInitArgs = term() %%% Result = [TestResult] | {error,Reason} %%% @doc Run tests as specified by the combination of options in <code>Opts</code>. %%% The options are the same as those used with the |