diff options
50 files changed, 4635 insertions, 140 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index 1a767a8197..a914dd0c19 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -52,7 +52,7 @@ CT_XML_FILES = $(CT_MODULES:=.xml) XML_APPLICATION_FILES = ref_man.xml XML_REF1_FILES = ct_run.xml -XML_REF3_FILES = $(CT_XML_FILES) +XML_REF3_FILES = $(CT_XML_FILES) ct_hooks.xml XML_REF6_FILES = common_test_app.xml XML_PART_FILES = part.xml @@ -71,6 +71,7 @@ XML_CHAPTER_FILES = \ cover_chapter.xml \ ct_master_chapter.xml \ event_handler_chapter.xml \ + ct_hooks_chapter.xml \ dependencies_chapter.xml \ notes.xml \ notes_history.xml diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index e30eef2488..b4e4b45d62 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -131,7 +131,8 @@ <type> <v> Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | - {silent_connections,Conns} | {stylesheet,CSSFile}</v> + {silent_connections,Conns} | {stylesheet,CSSFile} | + {ct_hooks, CTHs}</v> <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()}</v> <v> MilliSec = integer()</v> @@ -143,6 +144,9 @@ <v> UserData = term()</v> <v> Conns = [atom()]</v> <v> CSSFile = string()</v> + <v> CTHs = [CTHModule | {CTHModule, CTHInitArgs}]</v> + <v> CTHModule = atom()</v> + <v> CTHInitArgs = term()</v> </type> <desc> @@ -170,6 +174,10 @@ <p>With <c>userdata</c>, it is possible for the user to specify arbitrary test suite related information which can be read by calling <c>ct:userdata/2</c>.</p> + + <p>The <c>ct_hooks</c> tag specifies which + <seealso marker="ct_hooks_chapter">Common Test Hooks</seealso> + are to be run together with this suite.</p> <p>Other tuples than the ones defined will simply be ignored.</p> diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml new file mode 100644 index 0000000000..0d59ce3b22 --- /dev/null +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -0,0 +1,556 @@ +<?xml version="1.0" encoding="UTF-8" ?> + +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2010</year><year>2010</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Common Test Hooks</title> + <prepared>Lukas Larsson</prepared> + <responsible>Lukas Larsson</responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2010-12-02</date> + <rev>PA1</rev> + <file>ct_hooks.sgml</file> + </header> + <module>ct_hooks</module> + <modulesummary>A callback interface on top of Common Test</modulesummary> + + <description> + + <warning><p>This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.</p></warning> + + <p>The <em>Common Test Hook</em> (henceforth called CTH) framework allows + extensions of the default behaviour of Common Test by means of callbacks + before and after all test suite calls. It is meant for advanced users of + Common Test which want to abstract out behaviour which is common to + multiple test suites. </p> + + <p>In brief, Common Test Hooks allows you to:</p> + + <list> + <item>Manipulate the runtime config before each suite + configuration call</item> + <item>Manipulate the return of all suite configuration calls and in + extension the result of the test themselves.</item> + </list> + + <p>The following sections describe the mandatory and optional CTH + functions Common Test will call during test execution. For more details + see <seealso marker="ct_hooks_chapter">Common Test Hooks</seealso> in + the User's Guide.</p> + + <p>For information about how to add a CTH to your suite see + <seealso marker="ct_hooks_chapter#installing">Installing a CTH + </seealso> in the User's Guide.</p> + + <note><p>See the + <seealso marker="ct_hooks_chapter#example">Example CTH</seealso> + in the User's Guide for a minimal example of a CTH. </p></note> + + </description> + + <section> + <title>CALLBACK FUNCTIONS</title> + <p>The following functions define the callback interface + for a Common Test Hook.</p> + </section> + + <funcs> + <func> + <name>Module:init(Id, Opts) -> State</name> + <fsummary>Initiates the Common Test Hook</fsummary> + <type> + <v>Id = reference() | term()</v> + <v>Opts = term()</v> + <v>State = term()</v> + </type> + + <desc> + <p> MANDATORY </p> + + <p>Always called before any other callback function. + Use this to initiate any common state. + It should return a state for this CTH.</p> + + <p><c>Id</c> is the return value of + <seealso marker="#Module:id-1">id/1</seealso>, or a <c>reference</c> + (created using + <seealso marker="erts:erlang#make_ref-0">make_ref/0</seealso>) + if <seealso marker="#Module:id-1">id/1</seealso> is not implemented. + </p> + + <p>For details about when init is called see + <seealso marker="ct_hooks_chapter#scope">scope</seealso> + in the User's Guide.</p> + + </desc> + </func> + + <func> + <name>Module:pre_init_per_suite(SuiteName, Config, CTHState) -> + Result</name> + <fsummary>Called before init_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {Return, NewCTHState}</v> + <v>Return = NewConfig | SkipOrFail</v> + <v>SkipOrFail = {fail, Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> if it exists. + It typically contains initialization/logging which needs to be done + before init_per_suite is called. + If <c>{skip,Reason}</c> or <c>{fail,Reason}</c> is returned, + init_per_suite and all test cases of the suite will be skipped and + Reason printed in the overview log of the suite.</p> + + <p><c>SuiteName</c> is the name of the suite to be run.</p> + + <p><c>Config</c> is the original config list of the test suite.</p> + + <p><c>CTHState</c> is the current internal state of the CTH.</p> + + <p><c>Return</c> is the result of the init_per_suite function. + If it is <c>{skip,Reason}</c> or <c>{fail,Reason}</c> + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> will never be called, instead the initiation is considered + to be skipped/failed respectively. If a <c>NewConfig</c> list + is returned, <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> will be called with that <c>NewConfig</c> list. + See <seealso marker="ct_hooks_chapter#pre"> + Pre Hooks</seealso> in the User's Guide for more details.</p> + + + <p>Note that this function is only called if the CTH has been added + before init_per_suite is run, see + <seealso marker="ct_hooks_chapter#scope">CTH Scoping</seealso> + in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:post_init_per_suite(SuiteName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after init_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail, Reason} | {skip, Reason} | term()</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> if it exists. It typically contains extra + checks to make sure that all the correct dependencies have + been started correctly.</p> + + <p><c>Return</c> is what + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> returned, i.e. {fail,Reason}, {skip,Reason}, a <c>Config</c> + list or a term describing how + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> failed.</p> + + <p><c>NewReturn</c> is the possibly modified return value of + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso>. It is here possible to recover from a failure in + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> by returning the <c>ConfigList</c> with the <c>tc_status</c> + element removed. See <seealso marker="ct_hooks_chapter#post"> + Post Hooks</seealso> in the User's Guide for more details.</p> + + <p><c>CTHState</c> is the current internal state of the CTH.</p> + + <p>Note that this function is only called if the CTH has been added + before or in init_per_suite, see + <seealso marker="ct_hooks_chapter#scope">CTH Scoping</seealso> + in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:pre_init_per_group(GroupName, Config, CTHState) -> + Result</name> + <fsummary>Called before init_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> instead.</p> + </desc> + </func> + + <func> + <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after init_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_init_per_testcase(TestcaseName, Config, CTHState) -> + Result</name> + <fsummary>Called before init_per_testcase</fsummary> + <type> + <v>TestcaseName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_testcase-2"> + init_per_testcase</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_testcase-2"> + init_per_testcase</seealso> function instead.</p> + + <p>Note that it is not possible to add CTH's here right now, + that feature might be added later, + but it would right now break backwards compatability.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) + -> Result</name> + <fsummary>Called after end_per_testcase</fsummary> + <type> + <v>TestcaseName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_testcase-2"> + end_per_testcase</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_testcase-2"> + end_per_testcase</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_end_per_group(GroupName, Config, CTHState) -> + Result</name> + <fsummary>Called before end_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after end_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_end_per_suite(SuiteName, Config, CTHState) -> + Result</name> + <fsummary>Called before end_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_suite-2"> + end_per_suite</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_suite(SuiteName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after end_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_suite-2"> + end_per_suite</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:on_tc_fail(TestcaseName, Reason, CTHState) -> + NewCTHState</name> + <fsummary>Called after the CTH scope ends</fsummary> + <type> + <v>TestcaseName = init_per_suite | end_per_suite | + init_per_group | end_per_group | atom()</v> + <v>Reason = term()</v> + <v>CTHState = NewCTHState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called whenever a testcase fails. + It is called after the post function has been called for + the testcase which failed. i.e. + if init_per_suite fails this function is called after + <seealso marker="#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, and if a testcase fails it is called + after <seealso marker="#Module:post_end_per_testcase-4"> + post_end_per_testcase</seealso>.</p> + + <p>The data which comes with the Reason follows the same format as the + <seealso marker="event_handler_chapter#failreason">FailReason + </seealso> in the <seealso marker="event_handler_chapter#tc_done">tc_done</seealso> event. + See <seealso marker="event_handler_chapter#events">Event Handling + </seealso> in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:on_tc_skip(TestcaseName, Reason, CTHState) -> + NewCTHState</name> + <fsummary>Called after the CTH scope ends</fsummary> + <type> + <v>TestcaseName = end_per_suite | init_per_group | + end_per_group | atom()</v> + <v>Reason = {tc_auto_skip | tc_user_skip, term()}</v> + <v>CTHState = NewCTHState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called whenever a testcase is skipped. + It is called after the post function has been called for the + testcase which was skipped. + i.e. if init_per_group is skipped this function is called after + <seealso marker="#Module:post_init_per_suite-4">post_init_per_group + </seealso>, and if a testcase is skipped it is called after + <seealso marker="#Module:post_end_per_testcase-4">post_end_per_testcase + </seealso>.</p> + + <p>The data which comes with the Reason follows the same format as + <seealso marker="event_handler_chapter#tc_auto_skip">tc_auto_skip + </seealso> and <seealso marker="event_handler_chapter#tc_user_skip"> + tc_user_skip</seealso> events. + See <seealso marker="event_handler_chapter#events">Event Handling + </seealso> in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:terminate(CTHState)</name> + <fsummary>Called after the CTH scope ends</fsummary> + <type> + <v>CTHState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called at the end of a CTH's + <seealso marker="ct_hooks_chapter#scope">scope</seealso>. + </p> + </desc> + </func> + + <func> + <name>Module:id(Opts) -> Id</name> + <fsummary>Called before the init function of a CTH</fsummary> + <type> + <v>Opts = term()</v> + <v>Id = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>The <c>Id</c> is used to uniquely identify a CTH instance, + if two CTH's return the same <c>Id</c> the second CTH is ignored + and subsequent calls to the CTH will only be made to the first + instance. For more information see + <seealso marker="ct_hooks_chapter#installing">Installing a CTH + </seealso> in the User's Guide. + </p> + + <p>This function should NOT have any side effects as it might + be called multiple times by Common Test.</p> + + <p>If not implemented the CTH will act as if this function returned a + call to <c>make_ref/0</c>.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml new file mode 100644 index 0000000000..fc5ab48e1b --- /dev/null +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -0,0 +1,401 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2011</year><year>2011</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Common Test Hooks</title> + <prepared>Lukas Larsson</prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>ct_hooks_chapter.xml</file> + </header> + + <marker id="general"></marker> + <section> + <title>General</title> + <warning><p>This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.</p></warning> + <p> + The <em>Common Test Hook</em> (henceforth called CTH) framework allows + extensions of the default behaviour of Common Test by means of hooks + before and after all test suite calls. CTHs allow advanced Common Test + users to abstract out behaviour which is common to multiple test suites + without littering all test suites with library calls. Some example + usages are: logging, starting and monitoring external systems, + building C files needed by the tests and much more!</p> + + <p>In brief, Common Test Hooks allows you to:</p> + + <list> + <item>Manipulate the runtime config before each suite + configuration call</item> + <item>Manipulate the return of all suite configuration calls and in + extension the result of the test themselves.</item> + </list> + + <p>The following sections describe how to use CTHs, when they are run + and how to manipulate your test results in a CTH</p> + + <warning><p>When executing within a CTH all timetraps are shutoff. So + if your CTH never returns, the entire test run will be stalled!</p> + </warning> + + </section> + + <marker id="installing"></marker> + <section> + <title>Installing a CTH</title> + <p>There are multiple ways to install a CTH in your test run. You can do it + for all tests in a run, for specific test suites and for specific groups + within a test suite. If you want a CTH to be present in all test suites + within your test run there are three different ways to accomplish that. + </p> + + <list> + <item>Add <c>-ct_hooks</c> as an argument to + <seealso marker="run_test_chapter#ct_run">ct_run</seealso>. + To add multiple CTHs using this method append them to each other + using the keyword <c>and</c>, i.e. + <c>ct_run -ct_hooks cth1 [{debug,true}] and cth2 ...</c>.</item> + <item>Add the <c>ct_hooks</c> tag to your + <seealso marker="run_test_chapter#test_specifications"> + Test Specification</seealso></item> + <item>Add the <c>ct_hooks</c> tag to your call to + <seealso marker="ct#run_test-1">ct:run_test/1</seealso></item> + </list> + + <p>You can also add CTHs within a test suite. This is done by returning + <c>{ct_hooks,[CTH]}</c> in the config list from + <seealso marker="common_test#Module:suite-0">suite/0</seealso>, + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite/1</seealso> or + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group/2</seealso>. <c>CTH</c> in this case can be either + only the module name of the CTH or a tuple with the module name and the + initial arguments to the CTH. Eg: + <c>{ct_hooks,[my_cth_module]}</c> or + <c>{ct_hooks,[{my_cth_module,[{debug,true}]}]}</c></p> + + <section> + <title>Overriding CTHs</title> + <p>By default each installation of a CTH will cause a new instance of it + to be activated. This can cause problems if you want to be able to + override CTHs in test specifications while still having them in the + suite info function. The + <seealso marker="ct_hooks#Module:id-1">id/1</seealso> + callback exists to address this problem. By returning the same + <c>id</c> in both places, Common Test knows that this CTH + has already been installed and will not try to install it again.</p> + </section> + + </section> + + <marker id="scope"/> + <section> + <title>CTH Scope</title> + <p>Once the CTH is installed into a certain test run it will be there until + its scope is expired. The scope of a CTH depends on when it is + installed. + The <seealso marker="ct_hooks#Module:init-2">init/2</seealso> is + called at the beginning of the scope and the + <seealso marker="ct_hooks#Module:terminate-1">terminate/1 + </seealso> function is called when the scope ends.</p> + <table> + <row> + <cell><em>CTH Installed in</em></cell> + <cell><em>CTH scope begins before</em></cell> + <cell><em>CTH scope ends after</em></cell> + </row> + <row> + <cell><seealso marker="run_test_chapter#ct_run">ct_run</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="ct#run_test-1">ct:run_test</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="run_test_chapter#test_specifications"> + Test Specification</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:suite-0">suite/0 + </seealso></cell> + <cell><seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite/3</seealso> is called.</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> + post_end_per_suite/4</seealso> has been called for that test suite.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite/1</seealso></cell> + <cell><seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite/4</seealso> is called.</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> + post_end_per_suite/4</seealso> has been called for that test suite.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:init_per_group-2"> + init_per_group/2</seealso></cell> + <cell><seealso marker="ct_hooks#Module:post_init_per_group-4"> + post_init_per_group/4</seealso> is called.</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> + post_end_per_group/4</seealso> has been called for that group.</cell> + </row> + <tcaption>Scope of a CTH</tcaption> + </table> + + <section> + <title>CTH Processes and Tables</title> + <p>CTHs are run with the same process scoping as normal test suites + i.e. a different process will execute the init_per_suite hooks then the + init_per_group or per_testcase hooks. So if you want to spawn a + process in the CTH you cannot link with the CTH process as it will exit + after the post hook ends. Also if you for some reason need an ETS + table with your CTH, you will have to spawn a process which handles + it.</p> + </section> + + </section> + + <marker id="manipulating"/> + <section> + <title>Manipulating tests</title> + <p>It is through CTHs possible to manipulate the results of tests and + configuration functions. The main purpose of doing this with CTHs is to + allow common patterns to be abstracted out from test test suites and applied to + multiple test suites without duplicating any code. All of the callback + functions for a CTH follow a common interface, this interface is + described below.</p> + + <p>It is only possible to hook into test function which exists in the test + suite. So in order for a CTH to hook in before + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + the <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso> + function must exist in the test suite.</p> + + <marker id="pre"/> + <section> + <title>Pre Hooks</title> + <p> + It is possible in a CTH to hook in behaviour before + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_testcase</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and + <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>. + This is done in the CTH functions called pre_<name of function>. + All of these functions take the same three arguments: <c>Name</c>, + <c>Config</c> and <c>CTHState</c>. The return value of the CTH function + is always a combination of an result for the suite/group/test and an + updated <c>CTHState</c>. If you want the test suite to continue on + executing you should return the config list which you want the test to + use as the result. If you for some reason want to skip/fail the test, + return a tuple with <c>skip</c> or <c>fail</c> and a reason as the + result. Example: + </p> + <code>pre_init_per_suite(SuiteName, Config, CTHState) -> + case db:connect() of + {error,_Reason} -> + {{fail, "Could not connect to DB"}, CTHState}; + {ok, Handle} -> + {[{db_handle, Handle} | Config], CTHState#state{ handle = Handle }} + end.</code> + + </section> + + <marker id="post"/> + <section> + <title>Post Hooks</title> + <p>It is also possible in a CTH to hook in behaviour after + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_testcase</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and + <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>. + This is done in the CTH functions called post_<name of function>. + All of these function take the same four arguments: <c>Name</c>, + <c>Config</c>, <c>Return</c> and <c>CTHState</c>. <c>Config</c> in this + case is the same <c>Config</c> as the testcase is called with. + <c>Return</c> is the value returned by the testcase. If the testcase + failed by crashing, <c>Return</c> will be + <c>{'EXIT',{{Error,Reason},Stacktrace}}</c>.</p> + + <p>The return value of the CTH function is always a combination of an + result for the suite/group/test and an updated <c>CTHState</c>. If + you want the callback to not affect the outcome of the test you should + return the <c>Return</c> data as it is given to the CTH. You can also + modify the result of the test. By returning the <c>Config</c> list + with the <c>tc_status</c> element removed you can recover from a test + failure. As in all the pre hooks, it is also possible to fail/skip + the test case in the post hook. Example: </p> + + <code>post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + case db:check_consistency() of + true -> + %% DB is good, pass the test. + {proplists:delete(tc_status, Config), CTHState}; + false -> + %% DB is not good, mark as skipped instead of failing + {{skip, "DB is inconsisten!"}, CTHState} + end; +post_end_per_testcase(_TC, Config, Return, CTHState) -> + %% Do nothing if tc does not crash. + {Return, CTHState}.</code> + + <note>Recovering from a testcase failure using CTHs should only be done as + a last resort. If used wrongly it could become very difficult to + determine which tests pass or fail in a test run</note> + + </section> + + <marker id="skip_n_fail"/> + <section> + <title>Skip and Fail hooks</title> + <p> + After any post hook has been executed for all installed CTHs, + <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso> + or <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_skip</seealso> + might be called if the testcase failed or was skipped + respectively. You cannot affect the outcome of the tests any further at + this point. + </p> + </section> + + </section> + + <marker id="example"/> + <section> + <title>Example CTH</title> + <p>The CTH below will log information about a test run into a format + parseable by <seealso marker="kernel:file#consult-1">file:consult/1</seealso>. + </p> + <code>%%% @doc Common Test Example Common Test Hook module. +-module(example_cth). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3]). + +-export([terminate/1]). + +-record(state, { file_handle, total, suite_total, ts, tcs, data }). + +%% @doc Return a unique id for this CTH. +id(Opts) -> + proplists:get_value(filename, Opts, "/tmp/file.log"). + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, Opts) -> + {ok,D} = file:open(Id,[write]), + #state{ file_handle = D, total = 0, data = [] }. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{ suite_total = 0, tcs = [] }}. + +%% @doc Called after init_per_suite. +post_init_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(Suite,Config,Return,State) -> + Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)}, + {Return, State#state{ data = [Data | State#state.data] , + total = State#state.total + State#state.suite_total } }. + +%% @doc Called before each init_per_group. +pre_init_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called before each test case. +pre_init_per_testcase(TC,Config,State) -> + {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. + +%% @doc Called after each test case. +post_end_per_testcase(TC,Config,Return,State) -> + TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(TC, Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. +on_tc_skip(TC, Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(State) -> + io:format(State#state.file_handle, "~p.~n", + [{test_run, State#state.total, State#state.data}]), + file:close(State#state.file_handle), + ok.</code> + </section> + +</chapter> + + + + diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index 904876ac46..a01feb59d1 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -61,6 +61,7 @@ itself.</p> </section> <section> + <marker id="usage"></marker> <title>Usage</title> <p>Event handlers may be installed by means of an <c>event_handler</c> start flag (<c>ct_run</c>) or option (<c>ct:run_test/1</c>), where the @@ -120,6 +121,7 @@ node the event has originated from (only relevant for CT Master event handlers). <c>data</c> is specific for the particular event.</p> + <marker id="events"></marker> <p><em>General events:</em></p> <list> @@ -172,6 +174,7 @@ are also given. </p></item> + <marker id="tc_done"/> <item><c>#event{name = tc_done, data = {Suite,FuncOrGroup,Result}}</c> <p><c>Suite = atom()</c>, name of the suite.</p> <p><c>FuncOrGroup = Func | {Conf,GroupName,GroupProperties}</c></p> @@ -181,12 +184,14 @@ (unknown if init- or end function times out).</p> <p><c>GroupProperties = list()</c>, list of execution properties for the group.</p> <p><c>Result = ok | {skipped,SkipReason} | {failed,FailReason}</c>, the result.</p> + <marker id="skipreason"/> <p><c>SkipReason = {require_failed,RequireInfo} | {require_failed_in_suite0,RequireInfo} | {failed,{Suite,init_per_testcase,FailInfo}} | UserTerm</c>, the reason why the case has been skipped.</p> - <p><c>FailReason = {error,FailInfo} | + <marker id="failreason"/> + <p><c>FailReason = {error,FailInfo} | {error,{RunTimeError,StackTrace}} | {timetrap_timeout,integer()} | {failed,{Suite,end_per_testcase,FailInfo}}</c>, reason for failure.</p> @@ -209,6 +214,7 @@ <c>end_per_testcase</c> for the case failed. </p></item> + <marker id="tc_auto_skip"></marker> <item><c>#event{name = tc_auto_skip, data = {Suite,Func,Reason}}</c> <p><c>Suite = atom()</c>, the name of the suite.</p> <p><c>Func = atom()</c>, the name of the test case or configuration function.</p> @@ -234,7 +240,8 @@ skipped because of <c>init_per_testcase</c> failing, since that information is carried with the <c>tc_done</c> event. </p></item> - + + <marker id="tc_user_skip"></marker> <item><c>#event{name = tc_user_skip, data = {Suite,TestCase,Comment}}</c> <p><c>Suite = atom()</c>, name of the suite.</p> <p><c>TestCase = atom()</c>, name of the test case.</p> diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 53a4cb1bbf..41371b60be 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -75,6 +75,7 @@ <xi:include href="ct_master_chapter.xml"/> <xi:include href="event_handler_chapter.xml"/> <xi:include href="dependencies_chapter.xml"/> + <xi:include href="ct_hooks_chapter.xml"/> <xi:include href="why_test_chapter.xml"/> </part> diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index d5985bb021..631e3871c2 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -76,6 +76,7 @@ <xi:include href="ct_telnet.xml"/> <xi:include href="unix_telnet.xml"/> <xi:include href="ct_slave.xml"/> + <xi:include href="ct_hooks.xml"/> </application> diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 94fcf6bf01..7b485bdc35 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -105,6 +105,7 @@ RPC from a remote node.</p> </section> + <marker id="ct_run"></marker> <section> <title>Running tests from the OS command line</title> @@ -147,6 +148,8 @@ <seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item> <item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install <seealso marker="event_handler_chapter#event_handling">event handlers</seealso> including start arguments.</item> + <item><c><![CDATA[-ct_hooks <ct_hooks>]]></c>, to install + <seealso marker="ct_hooks_chapter#installing">Common Test Hooks</seealso> including start arguments.</item> <item><c><![CDATA[-include]]></c>, specifies include directories (see above).</item> <item><c><![CDATA[-no_auto_compile]]></c>, disables the automatic test suite compilation feature (see above).</item> <item><c><![CDATA[-multiply_timetraps <n>]]></c>, extends <seealso marker="write_test_chapter#timetraps">timetrap @@ -333,8 +336,8 @@ with <c>dir</c>.</p> </section> + <marker id="test_specifications"></marker> <section> - <marker id="test_specifications"></marker> <title>Using test specifications</title> <p>The most flexible way to specify what to test, is to use a so @@ -440,6 +443,9 @@ {event_handler, NodeRefs, EventHandlers}. {event_handler, EventHandlers, InitArgs}. {event_handler, NodeRefs, EventHandlers, InitArgs}. + + {ct_hooks, CTHModules}. + {ct_hooks, NodeRefs, CTHModules}. </pre> <p>Test terms:</p> <pre> @@ -478,6 +484,9 @@ LogDir = string() EventHandlers = atom() | [atom()] InitArgs = [term()] + CTHModules = [CTHModule | {CTHModule, CTHInitArgs}] + CTHModule = atom() + CTHInitArgs = term() DirRef = DirAlias | Dir Suites = atom() | [atom()] | all Suite = atom() diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 5afec6de6a..76493d3616 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -115,6 +115,7 @@ </p> </section> + <marker id="per_testcase"/> <section> <title>Init and end per test case</title> diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 027667e6b0..378a7ba08c 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -67,7 +67,9 @@ MODULES= \ ct_config \ ct_config_plain \ ct_config_xml \ - ct_slave + ct_slave \ + ct_hooks\ + ct_hooks_lock TARGET_MODULES= $(MODULES:%=$(EBIN)/%) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 405dc40c8b..b0a92dcc15 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -148,7 +148,8 @@ run(TestDirs) -> %%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} | %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | -%%% {refresh_logs,LogDir} | {basic_html,Bool} +%%% {refresh_logs,LogDir} | {basic_html,Bool} | +%%% {ct_hooks, CTHs} %%% TestDirs = [string()] | string() %%% Suites = [string()] | string() %%% Cases = [atom()] | atom() @@ -176,6 +177,9 @@ run(TestDirs) -> %%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile} %%% DecryptKey = string() %%% DecryptFile = string() +%%% CTHs = [CTHModule | {CTHModule, CTHInitArgs}] +%%% CTHModule = atom() +%%% CTHInitArgs = term() %%% Result = [TestResult] | {error,Reason} %%% @doc Run tests as specified by the combination of options in <code>Opts</code>. %%% The options are the same as those used with the diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index f2ca023cff..04829004f4 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -24,7 +24,7 @@ -module(ct_framework). --export([init_tc/3, end_tc/3, get_suite/2, report/2, warn/1]). +-export([init_tc/3, end_tc/4, get_suite/2, report/2, warn/1]). -export([error_notification/4]). -export([overview_html_header/1]). @@ -207,7 +207,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; - FinalConfig -> + {ok, FinalConfig} -> case MergeResult of {error,Reason} -> %% suite0 configure finished now, report that @@ -216,13 +216,25 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> _ -> case get('$test_server_framework_test') of undefined -> - FinalConfig; + ct_suite_init(Mod, FuncSpec, FinalConfig); Fun -> - Fun(init_tc, FinalConfig) + case Fun(init_tc, FinalConfig) of + NewConfig when is_list(NewConfig) -> + {ok,NewConfig}; + Else -> + Else + end end end end. - + +ct_suite_init(Mod, Func, [Config]) when is_list(Config) -> + case ct_hooks:init_tc( Mod, Func, Config) of + NewConfig when is_list(NewConfig) -> + {ok, [NewConfig]}; + Else -> + Else + end. add_defaults(Mod,Func,FuncInfo,DoInit) -> case (catch Mod:suite()) of @@ -239,7 +251,9 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> (_) -> false end, SuiteInfo) of true -> - SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo), + SuiteInfoNoCTH = + lists:keydelete(ct_hooks,1,SuiteInfo), + SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfoNoCTH), case add_defaults1(Mod,Func,FuncInfo,SuiteInfo1,DoInit) of Error = {error,_} -> {SuiteInfo1,Error}; MergedInfo -> {SuiteInfo1,MergedInfo} @@ -362,6 +376,8 @@ configure([{timetrap,off}|Rest],Info,SuiteInfo,Scope,Config) -> configure([{timetrap,Time}|Rest],Info,SuiteInfo,Scope,Config) -> Dog = test_server:timetrap(Time), configure(Rest,Info,SuiteInfo,Scope,[{watchdog,Dog}|Config]); +configure([{ct_hooks, Hook} | Rest], Info, SuiteInfo, Scope, Config) -> + configure(Rest, Info, SuiteInfo, Scope, [{ct_hooks, Hook} | Config]); configure([_|Rest],Info,SuiteInfo,Scope,Config) -> configure(Rest,Info,SuiteInfo,Scope,Config); configure([],_,_,_,Config) -> @@ -418,14 +434,14 @@ try_set_default(Name,Key,Info,Where) -> %%% %%% @doc Test server framework callback, called by the test_server %%% when a test case is finished. -end_tc(?MODULE,error_in_suite,_) -> % bad start! +end_tc(?MODULE,error_in_suite,_, _) -> % bad start! ok; -end_tc(Mod,Func,{TCPid,Result,[Args]}) when is_pid(TCPid) -> - end_tc(Mod,Func,TCPid,Result,Args); -end_tc(Mod,Func,{Result,[Args]}) -> - end_tc(Mod,Func,self(),Result,Args). +end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) -> + end_tc(Mod,Func,TCPid,Result,Args,Return); +end_tc(Mod,Func,{Result,[Args]}, Return) -> + end_tc(Mod,Func,self(),Result,Args,Return). -end_tc(Mod,Func,TCPid,Result,Args) -> +end_tc(Mod,Func,TCPid,Result,Args,Return) -> case lists:keysearch(watchdog,1,Args) of {value,{watchdog,Dog}} -> test_server:timetrap_cancel(Dog); false -> ok @@ -448,8 +464,10 @@ end_tc(Mod,Func,TCPid,Result,Args) -> {_,GroupName,_Props} = Group -> case lists:keysearch(save_config,1,Args) of {value,{save_config,SaveConfig}} -> - ct_util:save_suite_data(last_saved_config, - {Mod,{group,GroupName}},SaveConfig), + ct_util:save_suite_data( + last_saved_config, + {Mod,{group,GroupName}}, + SaveConfig), Group; false -> Group @@ -466,12 +484,33 @@ end_tc(Mod,Func,TCPid,Result,Args) -> end, ct_util:reset_silent_connections(), - %% send sync notification so that event handlers may print - %% in the log file before it gets closed - ct_event:sync_notify(#event{name=tc_done, - node=node(), - data={Mod,FuncSpec,tag(Result)}}), - case Result of + case get('$test_server_framework_test') of + undefined -> + {FinalResult,FinalNotify} = + case ct_hooks:end_tc( + Mod, FuncSpec, Args, Result, Return) of + '$ct_no_change' -> + {FinalResult = ok,Result}; + FinalResult -> + {FinalResult,FinalResult} + end, + % send sync notification so that event handlers may print + % in the log file before it gets closed + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={Mod,FuncSpec, + tag_cth(FinalNotify)}}); + Fun -> + % send sync notification so that event handlers may print + % in the log file before it gets closed + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={Mod,FuncSpec,tag(Result)}}), + FinalResult = Fun(end_tc, Return) + end, + + + case FinalResult of {skip,{sequence_failed,_,_}} -> %% ct_logs:init_tc is never called for a skipped test case %% in a failing sequence, so neither should end_tc @@ -490,12 +529,7 @@ end_tc(Mod,Func,TCPid,Result,Args) -> _ -> ok end, - case get('$test_server_framework_test') of - undefined -> - ok; - Fun -> - Fun(end_tc, ok) - end. + FinalResult. %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | @@ -511,6 +545,21 @@ tag(E = testcase_aborted_or_killed) -> tag(Other) -> Other. +tag_cth({STag,Reason}) when STag == skip; STag == skipped -> + {skipped,Reason}; +tag_cth({fail, Reason}) -> + {failed, {error,Reason}}; +tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; + ETag == timetrap_timeout; + ETag == testcase_aborted -> + {failed,E}; +tag_cth(E = testcase_aborted_or_killed) -> + {failed,E}; +tag_cth(List) when is_list(List) -> + ok; +tag_cth(Other) -> + Other. + %%%----------------------------------------------------------------- %%% @spec error_notification(Mod,Func,Args,Error) -> ok %%% Mod = atom() @@ -1137,6 +1186,18 @@ report(What,Data) -> ok; tc_done -> {_Suite,Case,Result} = Data, + case Result of + {failed, _} -> + ct_hooks:on_tc_fail(What, Data); + {skipped,{failed,{_,init_per_testcase,_}}} -> + ct_hooks:on_tc_skip(tc_auto_skip, Data); + {skipped,{require_failed,_}} -> + ct_hooks:on_tc_skip(tc_auto_skip, Data); + {skipped,_} -> + ct_hooks:on_tc_skip(tc_user_skip, Data); + _Else -> + ok + end, case {Case,Result} of {init_per_suite,_} -> ok; @@ -1154,8 +1215,8 @@ report(What,Data) -> add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); - {_,{FailOrSkip,_Reason}} -> - add_to_stats(FailOrSkip) + {_,{SkipOrFail,_Reason}} -> + add_to_stats(SkipOrFail) end; tc_user_skip -> %% test case specified as skipped in testspec @@ -1163,6 +1224,7 @@ report(What,Data) -> ct_event:sync_notify(#event{name=tc_user_skip, node=node(), data=Data}), + ct_hooks:on_tc_skip(What, Data), add_to_stats(user_skipped); tc_auto_skip -> %% test case skipped because of error in init_per_suite @@ -1175,6 +1237,7 @@ report(What,Data) -> ct_event:sync_notify(#event{name=tc_auto_skip, node=node(), data=Data}), + ct_hooks:on_tc_skip(What, Data), if Case /= end_per_suite, Case /= end_per_group -> add_to_stats(auto_skipped); true -> diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl new file mode 100644 index 0000000000..77b7566d9e --- /dev/null +++ b/lib/common_test/src/ct_hooks.erl @@ -0,0 +1,305 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework test execution control module. +%%% +%%% <p>This module is a proxy for calling and handling common test hooks.</p> + +-module(ct_hooks). + +%% API Exports +-export([init/1]). +-export([init_tc/3]). +-export([end_tc/5]). +-export([terminate/1]). +-export([on_tc_skip/2]). +-export([on_tc_fail/2]). + +-type proplist() :: [{atom(),term()}]. + +%% If you change this, remember to update ct_util:look -> stop clause as well. +-define(config_name, ct_hooks). + +%% ------------------------------------------------------------------------- +%% API Functions +%% ------------------------------------------------------------------------- + +%% @doc Called before any suites are started +-spec init(State :: term()) -> ok | + {error, Reason :: term()}. +init(Opts) -> + call([{Hook, call_id, undefined} || Hook <- get_new_hooks(Opts)], + ok, init, []). + + +%% @doc Called after all suites are done. +-spec terminate(Hooks :: term()) -> + ok. +terminate(Hooks) -> + call([{HookId, fun call_terminate/3} || {HookId,_,_} <- Hooks], + ct_hooks_terminate_dummy, terminate, Hooks), + ok. + +%% @doc Called as each test case is started. This includes all configuration +%% tests. +-spec init_tc(Mod :: atom(), Func :: atom(), Args :: list()) -> + NewConfig :: proplist() | + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()}. +init_tc(ct_framework, _Func, Args) -> + Args; +init_tc(Mod, init_per_suite, Config) -> + Info = case catch proplists:get_value(ct_hooks, Mod:suite()) of + List when is_list(List) -> + [{ct_hooks,List}]; + _Else -> + [] + end, + call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]); +init_tc(Mod, end_per_suite, Config) -> + call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); +init_tc(Mod, {init_per_group, GroupName, Opts}, Config) -> + maybe_start_locker(Mod, GroupName, Opts), + call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); +init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> + call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); +init_tc(_Mod, TC, Config) -> + call(fun call_generic/3, Config, [pre_init_per_testcase, TC]). + +%% @doc Called as each test case is completed. This includes all configuration +%% tests. +-spec end_tc(Mod :: atom(), + Func :: atom(), + Args :: list(), + Result :: term(), + Resturn :: term()) -> + NewConfig :: proplist() | + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()} | + ok | '$ct_no_change'. +end_tc(ct_framework, _Func, _Args, Result, _Return) -> + Result; + +end_tc(Mod, init_per_suite, Config, _Result, Return) -> + call(fun call_generic/3, Return, [post_init_per_suite, Mod, Config], + '$ct_no_change'); + +end_tc(Mod, end_per_suite, Config, Result, _Return) -> + call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config], + '$ct_no_change'); + +end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> + call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], + '$ct_no_change'); + +end_tc(Mod, {end_per_group, GroupName, Opts}, Config, Result, _Return) -> + Res = call(fun call_generic/3, Result, + [post_end_per_group, GroupName, Config], '$ct_no_change'), + maybe_stop_locker(Mod, GroupName,Opts), + Res; + +end_tc(_Mod, TC, Config, Result, _Return) -> + call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], + '$ct_no_change'). + +on_tc_skip(How, {_Suite, Case, Reason}) -> + call(fun call_cleanup/3, {How, Reason}, [on_tc_skip, Case]). + +on_tc_fail(_How, {_Suite, Case, Reason}) -> + call(fun call_cleanup/3, Reason, [on_tc_fail, Case]). + +%% ------------------------------------------------------------------------- +%% Internal Functions +%% ------------------------------------------------------------------------- +call_id(Mod, Config, Meta) when is_atom(Mod) -> + call_id({Mod, []}, Config, Meta); +call_id({Mod, Opts}, Config, Scope) -> + Id = catch_apply(Mod,id,[Opts], make_ref()), + {Config, {Id, scope(Scope), {Mod, {Id,Opts}}}}. + +call_init({Mod,{Id,Opts}},Config,_Meta) -> + NewState = Mod:init(Id, Opts), + {Config, {Mod, NewState}}. + +call_terminate({Mod, State}, _, _) -> + catch_apply(Mod,terminate,[State], ok), + {[],{Mod,State}}. + +call_cleanup({Mod, State}, Reason, [Function | Args]) -> + NewState = catch_apply(Mod,Function, Args ++ [Reason, State], + State), + {Reason, {Mod, NewState}}. + +call_generic({Mod, State}, Value, [Function | Args]) -> + {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], + {Value,State}), + {NewValue, {Mod, NewState}}. + +%% Generic call function +call(Fun, Config, Meta) -> + maybe_lock(), + Hooks = get_hooks(), + Res = call([{HookId,Fun} || {HookId,_, _} <- Hooks] ++ + get_new_hooks(Config, Fun), + remove(?config_name,Config), Meta, Hooks), + maybe_unlock(), + Res. + +call(Fun, Config, Meta, NoChangeRet) when is_function(Fun) -> + case call(Fun,Config,Meta) of + Config -> NoChangeRet; + NewReturn -> NewReturn + end; + +call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> + try + {Config, {NewId, _, _} = NewHook} = call_id(Hook, Config, Meta), + {NewHooks, NewRest} = + case lists:keyfind(NewId, 1, Hooks) of + false when NextFun =:= undefined -> + {Hooks ++ [NewHook], + [{NewId, fun call_init/3} | Rest]}; + ExistingHook when is_tuple(ExistingHook) -> + {Hooks, Rest}; + _ -> + {Hooks ++ [NewHook], + [{NewId, fun call_init/3},{NewId,NextFun} | Rest]} + end, + call(NewRest, Config, Meta, NewHooks) + catch Error:Reason -> + Trace = erlang:get_stacktrace(), + ct_logs:log("Suite Hook","Failed to start a CTH: ~p:~p", + [Error,{Reason,Trace}]), + call([], {fail,"Failed to start CTH" + ", see the CT Log for details"}, Meta, Hooks) + end; +call([{HookId, Fun} | Rest], Config, Meta, Hooks) -> + try + {_,Scope,ModState} = lists:keyfind(HookId, 1, Hooks), + {NewConf, NewHookInfo} = Fun(ModState, Config, Meta), + NewCalls = get_new_hooks(NewConf, Fun), + NewHooks = lists:keyreplace(HookId, 1, Hooks, {HookId, Scope, NewHookInfo}), + call(NewCalls ++ Rest, remove(?config_name, NewConf), Meta, + terminate_if_scope_ends(HookId, Meta, NewHooks)) + catch throw:{error_in_cth_call,Reason} -> + call(Rest, {fail, Reason}, Meta, + terminate_if_scope_ends(HookId, Meta, Hooks)) + end; +call([], Config, _Meta, Hooks) -> + save_suite_data_async(Hooks), + Config. + +remove(Key,List) when is_list(List) -> + [Conf || Conf <- List, is_tuple(Conf) =:= false + orelse element(1, Conf) =/= Key]; +remove(_, Else) -> + Else. + +%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc +scope([pre_init_per_testcase, TC|_]) -> + [post_end_per_testcase, TC]; +scope([pre_init_per_group, GroupName|_]) -> + [post_end_per_group, GroupName]; +scope([post_init_per_group, GroupName|_]) -> + [post_end_per_group, GroupName]; +scope([pre_init_per_suite, SuiteName|_]) -> + [post_end_per_suite, SuiteName]; +scope([post_init_per_suite, SuiteName|_]) -> + [post_end_per_suite, SuiteName]; +scope(init) -> + none. + +terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] -> + terminate_if_scope_ends(HookId,[Function,Tag],Hooks); +terminate_if_scope_ends(HookId, Function, Hooks) -> + case lists:keyfind(HookId, 1, Hooks) of + {HookId, Function, _ModState} = Hook -> + terminate([Hook]), + lists:keydelete(HookId, 1, Hooks); + _ -> + Hooks + end. + +%% Fetch hook functions +get_new_hooks(Config, Fun) -> + lists:foldl(fun(NewHook, Acc) -> + [{NewHook, call_id, Fun} | Acc] + end, [], get_new_hooks(Config)). + +get_new_hooks(Config) when is_list(Config) -> + lists:flatmap(fun({?config_name, HookConfigs}) -> + HookConfigs; + (_) -> + [] + end, Config); +get_new_hooks(_Config) -> + []. + +save_suite_data_async(Hooks) -> + ct_util:save_suite_data_async(?config_name, Hooks). + +get_hooks() -> + ct_util:read_suite_data(?config_name). + +catch_apply(M,F,A, Default) -> + try + apply(M,F,A) + catch error:Reason -> + case erlang:get_stacktrace() of + %% Return the default if it was the CTH module which did not have the function. + [{M,F,A}|_] when Reason == undef -> + Default; + Trace -> + ct_logs:log("Suite Hook","Call to CTH failed: ~p:~p", + [error,{Reason,Trace}]), + throw({error_in_cth_call, + lists:flatten( + io_lib:format("~p:~p/~p CTH call failed", + [M,F,length(A)]))}) + end + end. + + +%% We need to lock around the state for parallel groups only. This is because +%% we will get several processes reading and writing the state for a single +%% cth at the same time. +maybe_start_locker(Mod,GroupName,Opts) -> + case lists:member(parallel,Opts) of + true -> + {ok, _Pid} = ct_hooks_lock:start({Mod,GroupName}); + false -> + ok + end. + +maybe_stop_locker(Mod,GroupName,Opts) -> + case lists:member(parallel,Opts) of + true -> + stopped = ct_hooks_lock:stop({Mod,GroupName}); + false -> + ok + end. + + +maybe_lock() -> + locked = ct_hooks_lock:request(). + +maybe_unlock() -> + unlocked = ct_hooks_lock:release(). diff --git a/lib/common_test/src/ct_hooks_lock.erl b/lib/common_test/src/ct_hooks_lock.erl new file mode 100644 index 0000000000..98794201bb --- /dev/null +++ b/lib/common_test/src/ct_hooks_lock.erl @@ -0,0 +1,132 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework test execution control module. +%%% +%%% <p>This module is a proxy for calling and handling locks in +%%% common test hooks.</p> + +-module(ct_hooks_lock). + +-behaviour(gen_server). + +%% API +-export([start/1, stop/1, request/0, release/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, { id, locked = false, requests = [] }). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Starts the server +start(Id) -> + case gen_server:start({local, ?SERVER}, ?MODULE, Id, []) of + {error,{already_started, Pid}} -> + {ok,Pid}; + Else -> + Else + end. + +stop(Id) -> + try + gen_server:call(?SERVER, {stop,Id}) + catch exit:{noproc,_} -> + stopped + end. + +request() -> + try + gen_server:call(?SERVER,{request,self()},infinity) + catch exit:{noproc,_} -> + locked + end. + +release() -> + try + gen_server:call(?SERVER,{release,self()}) + catch exit:{noproc,_} -> + unlocked + end. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%% @doc Initiates the server +init(Id) -> + {ok, #state{ id = Id }}. + +%% @doc Handling call messages +handle_call({stop,Id}, _From, #state{ id = Id, requests = Reqs } = State) -> + [gen_server:reply(Req, locker_stopped) || {Req,_ReqId} <- Reqs], + {stop, normal, stopped, State}; +handle_call({stop,_Id}, _From, State) -> + {reply, stopped, State}; +handle_call({request, Pid}, _From, #state{ locked = false, + requests = [] } = State) -> + Ref = monitor(process, Pid), + {reply, locked, State#state{ locked = {true, Pid, Ref}} }; +handle_call({request, Pid}, From, #state{ requests = Reqs } = State) -> + {noreply, State#state{ requests = Reqs ++ [{From,Pid}] }}; +handle_call({release, Pid}, _From, #state{ locked = {true, Pid, Ref}, + requests = []} = State) -> + demonitor(Ref,[flush]), + {reply, unlocked, State#state{ locked = false }}; +handle_call({release, Pid}, _From, + #state{ locked = {true, Pid, Ref}, + requests = [{NextFrom,NextPid}|Rest]} = State) -> + demonitor(Ref,[flush]), + gen_server:reply(NextFrom,locked), + NextRef = monitor(process, NextPid), + {reply,unlocked,State#state{ locked = {true, NextPid, NextRef}, + requests = Rest } }; +handle_call({release, _Pid}, _From, State) -> + {reply, not_locked, State}. + +%% @doc Handling cast messages +handle_cast(_Msg, State) -> + {noreply, State}. + +%% @doc Handling all non call/cast messages +handle_info({'DOWN',Ref,process,Pid,_}, + #state{ locked = {true, Pid, Ref}, + requests = [{NextFrom,NextPid}|Rest] } = State) -> + gen_server:reply(NextFrom, locked), + NextRef = monitor(process, NextPid), + {noreply,State#state{ locked = {true, NextPid, NextRef}, + requests = Rest } }. + +%% @doc This function is called by a gen_server when it is about to terminate. +terminate(_Reason, _State) -> + ok. + +%% @doc Convert process state when code is changed +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% ------------------------------------------------------------------------- +%% Internal Functions +%% ------------------------------------------------------------------------- diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index d0e6ba5fa6..f50a46a241 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -54,6 +54,7 @@ logdir, config = [], event_handlers = [], + ct_hooks = [], include = [], silent_connections, stylesheet, @@ -171,6 +172,7 @@ script_start1(Parent, Args) -> ([]) -> true end, false, Args), EvHandlers = event_handler_args2opts(Args), + CTHooks = ct_hooks_args2opts(Args), %% check flags and set corresponding application env variables @@ -234,6 +236,7 @@ script_start1(Parent, Args) -> StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover, logdir = LogDir, event_handlers = EvHandlers, + ct_hooks = CTHooks, include = IncludeDirs, silent_connections = SilentConns, stylesheet = Stylesheet, @@ -305,6 +308,10 @@ script_start2(StartOpts = #opts{vts = undefined, SpecStartOpts#opts.scale_timetraps), AllEvHs = merge_vals([StartOpts#opts.event_handlers, SpecStartOpts#opts.event_handlers]), + AllCTHooks = merge_vals( + [StartOpts#opts.ct_hooks, + SpecStartOpts#opts.ct_hooks]), + AllInclude = merge_vals([StartOpts#opts.include, SpecStartOpts#opts.include]), application:set_env(common_test, include, AllInclude), @@ -315,6 +322,7 @@ script_start2(StartOpts = #opts{vts = undefined, logdir = LogDir, config = SpecStartOpts#opts.config, event_handlers = AllEvHs, + ct_hooks = AllCTHooks, include = AllInclude, multiply_timetraps = MultTT, scale_timetraps = ScaleTT}} @@ -332,7 +340,8 @@ script_start2(StartOpts = #opts{vts = undefined, {error,no_testspec_specified}; {undefined,_} -> % no testspec used case check_and_install_configfiles(InitConfig, TheLogDir, - Opts#opts.event_handlers) of + Opts#opts.event_handlers, + Opts#opts.ct_hooks) of ok -> % go on read tests from start flags script_start3(Opts#opts{config=InitConfig, logdir=TheLogDir}, Args); @@ -343,7 +352,8 @@ script_start2(StartOpts = #opts{vts = undefined, %% merge config from start flags with config from testspec AllConfig = merge_vals([InitConfig, Opts#opts.config]), case check_and_install_configfiles(AllConfig, TheLogDir, - Opts#opts.event_handlers) of + Opts#opts.event_handlers, + Opts#opts.ct_hooks) of ok -> % read tests from spec {Run,Skip} = ct_testspec:prepare_tests(Terms, node()), do_run(Run, Skip, Opts#opts{config=AllConfig, @@ -358,7 +368,8 @@ script_start2(StartOpts, Args) -> InitConfig = ct_config:prepare_config_list(Args), LogDir = which(logdir, StartOpts#opts.logdir), case check_and_install_configfiles(InitConfig, LogDir, - StartOpts#opts.event_handlers) of + StartOpts#opts.event_handlers, + StartOpts#opts.ct_hooks) of ok -> % go on read tests from start flags script_start3(StartOpts#opts{config=InitConfig, logdir=LogDir}, Args); @@ -366,11 +377,12 @@ script_start2(StartOpts, Args) -> Error end. -check_and_install_configfiles(Configs, LogDir, EvHandlers) -> +check_and_install_configfiles(Configs, LogDir, EvHandlers, CTHooks) -> case ct_config:check_config_files(Configs) of false -> install([{config,Configs}, - {event_handler,EvHandlers}], LogDir); + {event_handler,EvHandlers}, + {ct_hooks,CTHooks}], LogDir); {value,{error,{nofile,File}}} -> {error,{cant_read_config_file,File}}; {value,{error,{wrong_config,Message}}}-> @@ -438,11 +450,13 @@ script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, script_start4(#opts{label = Label, shell = true, config = Config, event_handlers = EvHandlers, + ct_hooks = CTHooks, logdir = LogDir, testspecs = Specs}, _Args) -> %% label - used by ct_logs application:set_env(common_test, test_label, Label), - InstallOpts = [{config,Config},{event_handler,EvHandlers}], + InstallOpts = [{config,Config},{event_handler,EvHandlers}, + {ct_hooks, CTHooks}], if Config == [] -> ok; true -> @@ -508,6 +522,7 @@ script_usage() -> "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -526,6 +541,7 @@ script_usage() -> "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -664,6 +680,9 @@ run_test1(StartOpts) -> end, Hs)) end, + %% CT Hooks + CTHooks = get_start_opt(ct_hooks, value, [], StartOpts), + %% silent connections SilentConns = get_start_opt(silent_connections, fun(all) -> []; @@ -733,7 +752,9 @@ run_test1(StartOpts) -> Opts = #opts{label = Label, cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, - event_handlers = EvHandlers, include = Include, + event_handlers = EvHandlers, + ct_hooks = CTHooks, + include = Include, silent_connections = SilentConns, stylesheet = Stylesheet, multiply_timetraps = MultiplyTT, @@ -784,11 +805,16 @@ run_spec_file(Relaxed, SpecOpts#opts.event_handlers]), AllInclude = merge_vals([Opts#opts.include, SpecOpts#opts.include]), + + AllCTHooks = merge_vals([Opts#opts.ct_hooks, + SpecOpts#opts.ct_hooks]), + application:set_env(common_test, include, AllInclude), case check_and_install_configfiles(AllConfig, which(logdir,LogDir), - AllEvHs) of + AllEvHs, + AllCTHooks) of ok -> Opts1 = Opts#opts{label = Label, cover = Cover, @@ -798,7 +824,8 @@ run_spec_file(Relaxed, include = AllInclude, testspecs = AbsSpecs, multiply_timetraps = MultTT, - scale_timetraps = ScaleTT}, + scale_timetraps = ScaleTT, + ct_hooks = AllCTHooks}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); {error,GCFReason} -> @@ -808,10 +835,12 @@ run_spec_file(Relaxed, run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, config = CfgFiles, - event_handlers = EvHandlers}, + event_handlers = EvHandlers, + ct_hooks = CTHooks}, StartOpts) -> LogDir1 = which(logdir, LogDir), - case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of + case check_and_install_configfiles(CfgFiles, LogDir1, + EvHandlers, CTHooks) of ok -> reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1}, StartOpts)); @@ -842,7 +871,8 @@ check_config_file(Callback, File)-> run_dir(Opts = #opts{logdir = LogDir, config = CfgFiles, - event_handlers = EvHandlers}, StartOpts) -> + event_handlers = EvHandlers, + ct_hooks = CTHook }, StartOpts) -> LogDir1 = which(logdir, LogDir), Opts1 = Opts#opts{logdir = LogDir1}, AbsCfgFiles = @@ -863,7 +893,9 @@ run_dir(Opts = #opts{logdir = LogDir, check_config_file(Callback, File) end, FileList)} end, CfgFiles), - case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir1) of + case install([{config,AbsCfgFiles}, + {event_handler,EvHandlers}, + {ct_hooks, CTHook}], LogDir1) of ok -> ok; {error,IReason} -> exit(IReason) end, @@ -968,7 +1000,8 @@ run_testspec1(TestSpec) -> application:set_env(common_test, include, AllInclude), LogDir1 = which(logdir,Opts#opts.logdir), case check_and_install_configfiles(Opts#opts.config, LogDir1, - Opts#opts.event_handlers) of + Opts#opts.event_handlers, + Opts#opts.ct_hooks) of ok -> Opts1 = Opts#opts{testspecs = [], logdir = LogDir1, @@ -986,6 +1019,7 @@ get_data_for_node(#testspec{label = Labels, config = Cfgs, userconfig = UsrCfgs, event_handler = EvHs, + ct_hooks = CTHooks, include = Incl, multiply_timetraps = MTs, scale_timetraps = STs}, Node) -> @@ -1000,12 +1034,14 @@ get_data_for_node(#testspec{label = Labels, ConfigFiles = [{?ct_config_txt,F} || {N,F} <- Cfgs, N==Node] ++ [CBF || {N,CBF} <- UsrCfgs, N==Node], EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node], + FiltCTHooks = [Hook || {N,Hook} <- CTHooks, N==Node], Include = [I || {N,I} <- Incl, N==Node], #opts{label = Label, logdir = LogDir, cover = Cover, config = ConfigFiles, event_handlers = EvHandlers, + ct_hooks = FiltCTHooks, include = Include, multiply_timetraps = MT, scale_timetraps = ST}. @@ -1036,15 +1072,7 @@ refresh_logs(LogDir) -> which(logdir, undefined) -> "."; which(logdir, Dir) -> - Dir; -which(multiply_timetraps, undefined) -> - 1; -which(multiply_timetraps, MT) -> - MT; -which(scale_timetraps, undefined) -> - false; -which(scale_timetraps, ST) -> - ST. + Dir. choose_val(undefined, V1) -> V1; @@ -2032,12 +2060,37 @@ get_start_opt(Key, IfExists, IfNotExists, Args) -> Val; {value,{Key,_Val}} -> IfExists; - _ when is_function(IfNotExists) -> - IfNotExists(); _ -> IfNotExists end. +ct_hooks_args2opts(Args) -> + ct_hooks_args2opts( + proplists:get_value(ct_hooks, Args, []),[]). + +ct_hooks_args2opts([CTH,Arg,"and"| Rest],Acc) -> + ct_hooks_args2opts(Rest,[{list_to_atom(CTH), + parse_cth_args(Arg)}|Acc]); +ct_hooks_args2opts([CTH], Acc) -> + ct_hooks_args2opts([CTH,"and"],Acc); +ct_hooks_args2opts([CTH, "and" | Rest], Acc) -> + ct_hooks_args2opts(Rest,[list_to_atom(CTH)|Acc]); +ct_hooks_args2opts([CTH, Args], Acc) -> + ct_hooks_args2opts([CTH, Args, "and"],Acc); +ct_hooks_args2opts([],Acc) -> + lists:reverse(Acc). + +parse_cth_args(String) -> + try + true = io_lib:printable_list(String), + {ok,Toks,_} = erl_scan:string(String++"."), + {ok, Args} = erl_parse:parse_term(Toks), + Args + catch _:_ -> + String + end. + + event_handler_args2opts(Args) -> case proplists:get_value(event_handler, Args) of undefined -> @@ -2165,6 +2218,22 @@ opts2args(EnvStartOpts) -> end, EHs), [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), [{event_handler_init,lists:reverse(StrsR)}]; + ({ct_hooks,[]}) -> + []; + ({ct_hooks,CTHs}) when is_list(CTHs) -> + io:format(user,"ct_hooks: ~p",[CTHs]), + Strs = lists:flatmap( + fun({CTH,Arg}) -> + [atom_to_list(CTH), + lists:flatten( + io_lib:format("~p",[Arg])), + "and"]; + (CTH) when is_atom(CTH) -> + [atom_to_list(CTH),"and"] + end,CTHs), + [_LastAnd|StrsR] = lists:reverse(Strs), + io:format(user,"return: ~p",[lists:reverse(StrsR)]), + [{ct_hooks,lists:reverse(StrsR)}]; ({Opt,As=[A|_]}) when is_atom(A) -> [{Opt,[atom_to_list(Atom) || Atom <- As]}]; ({Opt,Strs=[S|_]}) when is_list(S) -> @@ -2263,12 +2332,19 @@ do_trace(Terms) -> dbg:tracer(), dbg:p(self(), [sos,call]), lists:foreach(fun({m,M}) -> - case dbg:tpl(M,[{'_',[],[{return_trace}]}]) of + case dbg:tpl(M,x) of + {error,What} -> exit({error,{tracing_failed,What}}); + _ -> ok + end; + ({me,M}) -> + case dbg:tp(M,[{'_',[],[{exception_trace}, + {message,{caller}}]}]) of {error,What} -> exit({error,{tracing_failed,What}}); _ -> ok end; ({f,M,F}) -> - case dbg:tpl(M,F,[{'_',[],[{return_trace}]}]) of + case dbg:tpl(M,F,[{'_',[],[{exception_trace}, + {message,{caller}}]}]) of {error,What} -> exit({error,{tracing_failed,What}}); _ -> ok end; diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index f5069427a2..2b6abefb72 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -394,8 +394,6 @@ filter_init_terms([Term|Ts], NewTerms, Spec)-> filter_init_terms([], NewTerms, Spec)-> {lists:reverse(NewTerms), Spec}. -add_option([], _, List, _)-> - List; add_option({Key, Value}, Node, List, WarnIfExists) when is_list(Value)-> OldOptions = case lists:keyfind(Node, 1, List) of {Node, Options}-> @@ -625,6 +623,20 @@ add_tests([{event_handler,Node,H,Args}|Ts],Spec) when is_atom(H) -> Node1 = ref2node(Node,Spec#testspec.nodes), add_tests(Ts,Spec#testspec{event_handler=[{Node1,H,Args}|EvHs]}); +%% --- ct_hooks -- +add_tests([{ct_hooks, all_nodes, Hooks} | Ts], Spec) -> + Tests = [{ct_hooks,N,Hooks} || N <- list_nodes(Spec)], + add_tests(Tests ++ Ts, Spec); +add_tests([{ct_hooks, Node, [Hook|Hooks]}|Ts], Spec) -> + SuiteCbs = Spec#testspec.ct_hooks, + Node1 = ref2node(Node,Spec#testspec.nodes), + add_tests([{ct_hooks, Node, Hooks} | Ts], + Spec#testspec{ct_hooks = [{Node1,Hook} | SuiteCbs]}); +add_tests([{ct_hooks, _Node, []}|Ts], Spec) -> + add_tests(Ts, Spec); +add_tests([{ct_hooks, Hooks}|Ts], Spec) -> + add_tests([{ct_hooks, all_nodes, Hooks}|Ts], Spec); + %% --- include --- add_tests([{include,all_nodes,InclDirs}|Ts],Spec) -> Tests = lists:map(fun(N) -> {include,N,InclDirs} end, list_nodes(Spec)), @@ -1051,6 +1063,8 @@ valid_terms() -> {event_handler,2}, {event_handler,3}, {event_handler,4}, + {ct_hooks,2}, + {ct_hooks,3}, {multiply_timetraps,2}, {multiply_timetraps,3}, {scale_timetraps,2}, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index b5ab4cbb6e..45146de57c 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -32,7 +32,9 @@ -export([close_connections/0]). --export([save_suite_data/3, save_suite_data/2, read_suite_data/1, +-export([save_suite_data/3, save_suite_data/2, + save_suite_data_async/3, save_suite_data_async/2, + read_suite_data/1, delete_suite_data/0, delete_suite_data/1, match_delete_suite_data/1, delete_testdata/0, delete_testdata/1, set_testdata/1, get_testdata/1, update_testdata/2]). @@ -159,6 +161,17 @@ do_start(Parent,Mode,LogDir) -> ok end, {StartTime,TestLogDir} = ct_logs:init(Mode), + + %% Initiate ct_hooks + case catch ct_hooks:init(Opts) of + ok -> + ok; + {_,CTHReason} -> + ct_logs:tc_print('Suite Callback',CTHReason,[]), + Parent ! {self(), CTHReason}, + self() ! {{stop,normal},{self(),make_ref()}} + end, + ct_event:notify(#event{name=test_start, node=node(), data={StartTime, @@ -182,12 +195,19 @@ read_opts() -> {error,{bad_installation,Error}} end. + save_suite_data(Key, Value) -> call({save_suite_data, {Key, undefined, Value}}). save_suite_data(Key, Name, Value) -> call({save_suite_data, {Key, Name, Value}}). +save_suite_data_async(Key, Value) -> + save_suite_data_async(Key, undefined, Value). + +save_suite_data_async(Key, Name, Value) -> + cast({save_suite_data, {Key, Name, Value}}). + read_suite_data(Key) -> call({read_suite_data, Key}). @@ -299,6 +319,10 @@ loop(Mode,TestData,StartDir) -> ct_event:sync_notify(#event{name=test_done, node=node(), data=Time}), + Callbacks = ets:lookup_element(?suite_table, + ct_hooks, + #suite_data.value), + ct_hooks:terminate(Callbacks), close_connections(ets:tab2list(?conn_table)), ets:delete(?conn_table), ets:delete(?board_table), @@ -308,6 +332,9 @@ loop(Mode,TestData,StartDir) -> ct_config:stop(), file:set_cwd(StartDir), return(From,ok); + {Ref, _Msg} when is_reference(Ref) -> + %% This clause is used when doing cast operations. + loop(Mode,TestData,StartDir); {get_mode,From} -> return(From,Mode), loop(Mode,TestData,StartDir); @@ -713,6 +740,9 @@ call(Msg) -> return({To,Ref},Result) -> To ! {Ref, Result}. +cast(Msg) -> + ct_util_server ! {Msg, {ct_util_server, make_ref()}}. + seconds(T) -> test_server:seconds(T). diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index ee973f6220..4ed29bdcd1 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -36,6 +36,7 @@ config=[], userconfig=[], event_handler=[], + ct_hooks=[], include=[], multiply_timetraps=[], scale_timetraps=[], diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index f2fe3390cf..be4b4c32b8 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -40,7 +40,8 @@ MODULES= \ ct_test_server_if_1_SUITE \ ct_config_SUITE \ ct_master_SUITE \ - ct_misc_1_SUITE + ct_misc_1_SUITE \ + ct_hooks_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl new file mode 100644 index 0000000000..1e187aa205 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -0,0 +1,1021 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_error_SUITE +%%% +%%% Description: +%%% Test various errors in Common Test suites. +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_hooks_SUITE). + +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir,"cth/tests/"), + CTHs = filelib:wildcard(filename:join(TestDir,"*_cth.erl")), + io:format("CTHs: ~p",[CTHs]), + [io:format("Compiling ~p: ~p", + [FileName,compile:file(FileName,[{outdir,TestDir},debug_info])]) || + FileName <- CTHs], + ct_test_support:init_per_suite([{path_dirs,[TestDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{seconds,20}}]. + +all() -> + all(suite). + +all(suite) -> + lists:reverse( + [ + one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, + faulty_cth_exit_in_init, faulty_cth_exit_in_id, + faulty_cth_exit_in_init_scope_suite, minimal_cth, + minimal_and_maximal_cth, faulty_cth_undef, + scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, + scope_per_suite_state_cth, scope_per_group_state_cth, + scope_suite_state_cth, + fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, + skip_post_suite_cth, recover_post_suite_cth, update_config_cth, + state_update_cth, options_cth, same_id_cth, + fail_n_skip_with_minimal_cth + ] + ) + . + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +one_cth(Config) when is_list(Config) -> + do_test(one_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth], Config). + +two_cth(Config) when is_list(Config) -> + do_test(two_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth,empty_cth], + Config). + +faulty_cth_no_init(Config) when is_list(Config) -> + do_test(faulty_cth_no_init, "ct_cth_empty_SUITE.erl",[askjhdkljashdkaj], + Config,{error,"Failed to start CTH, see the " + "CT Log for details"}). + +faulty_cth_id_no_init(Config) when is_list(Config) -> + do_test(faulty_cth_id_no_init, "ct_cth_empty_SUITE.erl",[id_no_init_cth], + Config,{error,"Failed to start CTH, see the " + "CT Log for details"}). + +minimal_cth(Config) when is_list(Config) -> + do_test(minimal_cth, "ct_cth_empty_SUITE.erl",[minimal_cth],Config). + +minimal_and_maximal_cth(Config) when is_list(Config) -> + do_test(minimal_and_maximal_cth, "ct_cth_empty_SUITE.erl", + [minimal_cth, empty_cth],Config). + +faulty_cth_undef(Config) when is_list(Config) -> + do_test(faulty_cth_undef, "ct_cth_empty_SUITE.erl", + [undef_cth],Config). + +faulty_cth_exit_in_init_scope_suite(Config) when is_list(Config) -> + do_test(faulty_cth_exit_in_init_scope_suite, + "ct_exit_in_init_scope_suite_cth_SUITE.erl", + [],Config). + +faulty_cth_exit_in_init(Config) when is_list(Config) -> + do_test(faulty_cth_exit_in_init, "ct_cth_empty_SUITE.erl", + [crash_init_cth], Config, + {error,"Failed to start CTH, see the " + "CT Log for details"}). + +faulty_cth_exit_in_id(Config) when is_list(Config) -> + do_test(faulty_cth_exit_in_id, "ct_cth_empty_SUITE.erl", + [crash_id_cth], Config, + {error,"Failed to start CTH, see the " + "CT Log for details"}). + +scope_per_suite_cth(Config) when is_list(Config) -> + do_test(scope_per_suite_cth, "ct_scope_per_suite_cth_SUITE.erl", + [],Config). + +scope_suite_cth(Config) when is_list(Config) -> + do_test(scope_suite_cth, "ct_scope_suite_cth_SUITE.erl", + [],Config). + +scope_per_group_cth(Config) when is_list(Config) -> + do_test(scope_per_group_cth, "ct_scope_per_group_cth_SUITE.erl", + [],Config). + +scope_per_suite_state_cth(Config) when is_list(Config) -> + do_test(scope_per_suite_state_cth, "ct_scope_per_suite_state_cth_SUITE.erl", + [],Config). + +scope_suite_state_cth(Config) when is_list(Config) -> + do_test(scope_suite_state_cth, "ct_scope_suite_state_cth_SUITE.erl", + [],Config). + +scope_per_group_state_cth(Config) when is_list(Config) -> + do_test(scope_per_group_state_cth, "ct_scope_per_group_state_cth_SUITE.erl", + [],Config). + +fail_pre_suite_cth(Config) when is_list(Config) -> + do_test(fail_pre_suite_cth, "ct_cth_empty_SUITE.erl", + [fail_pre_suite_cth],Config). + +fail_post_suite_cth(Config) when is_list(Config) -> + do_test(fail_post_suite_cth, "ct_cth_empty_SUITE.erl", + [fail_post_suite_cth],Config). + +skip_pre_suite_cth(Config) when is_list(Config) -> + do_test(skip_pre_suite_cth, "ct_cth_empty_SUITE.erl", + [skip_pre_suite_cth],Config). + +skip_post_suite_cth(Config) when is_list(Config) -> + do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl", + [skip_post_suite_cth],Config). + +recover_post_suite_cth(Config) when is_list(Config) -> + do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl", + [recover_post_suite_cth],Config). + +update_config_cth(Config) when is_list(Config) -> + do_test(update_config_cth, "ct_update_config_SUITE.erl", + [update_config_cth],Config). + +state_update_cth(Config) when is_list(Config) -> + do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl", + [state_update_cth,state_update_cth],Config). + +options_cth(Config) when is_list(Config) -> + do_test(options_cth, "ct_cth_empty_SUITE.erl", + [{empty_cth,[test]}],Config). + +same_id_cth(Config) when is_list(Config) -> + do_test(same_id_cth, "ct_cth_empty_SUITE.erl", + [same_id_cth,same_id_cth],Config). + +fail_n_skip_with_minimal_cth(Config) when is_list(Config) -> + do_test(fail_n_skip_with_minimal_cth, "ct_cth_fail_one_skip_one_SUITE.erl", + [minimal_terminate_cth],Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, SWC, CTHs, Config) -> + do_test(Tag, SWC, CTHs, Config, ok). +do_test(Tag, SWC, CTHs, Config, {error,_} = Res) -> + do_test(Tag, SWC, CTHs, Config, Res, 1); +do_test(Tag, SWC, CTHs, Config, Res) -> + do_test(Tag, SWC, CTHs, Config, Res, 2). + +do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> + + DataDir = ?config(data_dir, Config), + Suites = filelib:wildcard( + filename:join([DataDir,"cth/tests",SuiteWildCard])), + {Opts,ERPid} = setup([{suite,Suites}, + {ct_hooks,CTHs},{label,Tag}], Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). +%reformat(Events, _EH) -> +% Events. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(one_empty_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(two_empty_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_no_init) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_id_no_init) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {negative,{?eh,tc_start,'_'}, + {?eh,test_done,{'DEF','STOP_TIME'}}}, + {?eh,stop_logging,[]} + ]; + +test_events(minimal_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {negative,{?eh,cth,{'_',id,['_',[]]}}, + {?eh,cth,{'_',init,['_',[]]}}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(minimal_and_maximal_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {negative,{?eh,cth,{'_',id,['_',[]]}}, + {?eh,cth,{'_',init,['_',[]]}}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_undef) -> + FailReasonStr = "undef_cth:pre_init_per_suite/3 CTH call failed", + FailReason = {ct_cth_empty_SUITE,init_per_suite, + {failed,FailReasonStr}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, + {failed, {error,FailReasonStr}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, + {failed, FailReason}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,end_per_suite, + {failed, FailReason}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_exit_in_init_scope_suite) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',init_per_suite}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,tc_done, + {ct_exit_in_init_scope_suite_cth_SUITE,init_per_suite, + {failed, + {error, + "Failed to start CTH, see the CT Log for details"}}}}, + {?eh,tc_auto_skip, + {ct_exit_in_init_scope_suite_cth_SUITE,test_case, + {failed, + {ct_exit_in_init_scope_suite_cth_SUITE,init_per_suite, + {failed, + "Failed to start CTH, see the CT Log for details"}}}}}, + {?eh,tc_auto_skip, + {ct_exit_in_init_scope_suite_cth_SUITE,end_per_suite, + {failed, + {ct_exit_in_init_scope_suite_cth_SUITE,init_per_suite, + {failed, + "Failed to start CTH, see the CT Log for details"}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(faulty_cth_exit_in_init) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(faulty_cth_exit_in_id) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {negative, {?eh,tc_start,'_'}, + {?eh,test_done,{'DEF','STOP_TIME'}}}, + {?eh,stop_logging,[]}]; + +test_events(scope_per_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_per_suite_cth_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [ct_scope_per_suite_cth_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_per_suite_cth_SUITE,'$proplist','_',[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_suite_cth_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_suite_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_suite_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist','_',[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,tc_done,{ct_scope_suite_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_per_group_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}], + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_per_suite_state_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[test]]}}, + {?eh,cth,{'_',init,['_',[test]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_per_suite_state_cth_SUITE,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [ct_scope_per_suite_state_cth_SUITE,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_per_suite_state_cth_SUITE,'$proplist','_',[test]]}}, + {?eh,cth,{'_',terminate,[[test]]}}, + {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_suite_state_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[test]]}}, + {?eh,cth,{'_',init,['_',[test]]}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist',[test]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist','_',[test]]}}, + {?eh,cth,{'_',terminate,[[test]]}}, + {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_per_group_state_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]}}}, + {?eh,cth,{'_',id,[[test]]}}, + {?eh,cth,{'_',init,['_',[test]]}}, + {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}}, + {?eh,cth,{'_',terminate,[[test]]}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}], + + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(fail_pre_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + + + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist', + {fail,"Test failure"},[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, + {failed, {error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail, + [init_per_suite,{failed,"Test failure"},[]]}}, + + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, + {failed,{ct_cth_empty_SUITE,init_per_suite, + {failed,"Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip, + [test_case, {tc_auto_skip, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}},[]]}}, + + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip, + [end_per_suite, {tc_auto_skip, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}},[]]}}, + + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth, {'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(fail_post_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, + {failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,[init_per_suite, {failed,"Test failure"}, []]}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, + {failed,{ct_cth_empty_SUITE,init_per_suite, + {failed,"Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}}, + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,'_'},[]]}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth, {'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(skip_pre_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist',{skip,"Test skip"},[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, + {?eh,cth,{'_',on_tc_skip, + [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth, {'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(skip_post_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, + {?eh,cth,{'_',on_tc_skip, + [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(recover_post_suite_cth) -> + Suite = ct_cth_fail_per_suite_SUITE, + [ + {?eh,start_logging,'_'}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{Suite,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[Suite,'$proplist','$proplist']}}, + {?eh,cth,{'_',post_init_per_suite,[Suite,contains([tc_status]), + {'EXIT',{'_','_'}},[]]}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase, + [test_case, not_contains([tc_status]),[]]}}, + {?eh,cth,{'_',post_end_per_testcase, + [test_case, contains([tc_status]),'_',[]]}}, + {?eh,tc_done,{Suite,test_case,ok}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [Suite,not_contains([tc_status]),[]]}}, + {?eh,cth,{'_',post_end_per_suite, + [Suite,not_contains([tc_status]),'_',[]]}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(update_config_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + + {?eh,tc_start,{ct_update_config_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite, + [ct_update_config_SUITE,contains([]),[]]}}, + {?eh,cth,{'_',post_init_per_suite, + [ct_update_config_SUITE, + '$proplist', + contains( + [init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,tc_done,{ct_update_config_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_init_per_group, + [group1,contains( + [post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_init_per_group, + [group1, + contains( + [post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + contains( + [init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,tc_done,{ct_update_config_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_update_config_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase, + [test_case,contains( + [post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_end_per_testcase, + [test_case,contains( + [init_per_testcase, + pre_init_per_testcase, + post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + ok,[]]}}, + {?eh,tc_done,{ct_update_config_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group, + [group1,contains( + [post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_end_per_group, + [group1, + contains( + [pre_end_per_group, + post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + ok,[]]}}, + {?eh,tc_done,{ct_update_config_SUITE,{end_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_update_config_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [ct_update_config_SUITE,contains( + [post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_end_per_suite, + [ct_update_config_SUITE,contains( + [pre_end_per_suite, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + '_',[]]}}, + {?eh,tc_done,{ct_update_config_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[contains( + [post_end_per_suite, + pre_end_per_suite, + post_init_per_suite, + init_per_suite, + pre_init_per_suite])]}}, + {?eh,stop_logging,[]} + ]; + +test_events(state_update_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',init_per_suite}}, + + {?eh,tc_done,{'_',end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[contains( + [post_end_per_suite,pre_end_per_suite, + post_end_per_group,pre_end_per_group, + {not_in_order, + [post_end_per_testcase,pre_init_per_testcase, + on_tc_skip,post_end_per_testcase, + pre_init_per_testcase,on_tc_fail, + post_end_per_testcase,pre_init_per_testcase] + }, + post_init_per_group,pre_init_per_group, + post_init_per_suite,pre_init_per_suite, + init])]}}, + {?eh,cth,{'_',terminate,[contains( + [post_end_per_suite,pre_end_per_suite, + post_end_per_group,pre_end_per_group, + {not_in_order, + [post_end_per_testcase,pre_init_per_testcase, + on_tc_skip,post_end_per_testcase, + pre_init_per_testcase,on_tc_fail, + post_end_per_testcase,pre_init_per_testcase] + }, + post_init_per_group,pre_init_per_group, + post_init_per_suite,pre_init_per_suite, + init] + )]}}, + {?eh,stop_logging,[]} + ]; + +test_events(options_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,init,['_',[test]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_cth_empty_SUITE,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_cth_empty_SUITE,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[test]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[test]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(same_id_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,[same_id_cth,[]]}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {negative, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}}, + {negative, + {?eh,cth,{'_',post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {negative, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}}, + {negative, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {negative, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}}, + {negative, + {?eh,cth,{'_',post_end_per_suite, + [ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(fail_n_skip_with_minimal_cth) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',init_per_suite}}, + + {?eh,tc_done,{'_',end_per_suite,ok}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(ok) -> + ok. + + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_id_cth.erl new file mode 100644 index 0000000000..02c36e378c --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_id_cth.erl @@ -0,0 +1,34 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(crash_id_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([id/1]).
+
+id(Opts) ->
+ empty_cth:id(Opts),
+ exit(diediedie).
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_init_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_init_cth.erl new file mode 100644 index 0000000000..6ed23565f6 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_init_cth.erl @@ -0,0 +1,34 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(crash_init_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([init/2]).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts),
+ exit(diediedie).
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_empty_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_empty_SUITE.erl new file mode 100644 index 0000000000..499069b382 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_empty_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_empty_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_one_skip_one_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_one_skip_one_SUITE.erl new file mode 100644 index 0000000000..017812c719 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_one_skip_one_SUITE.erl @@ -0,0 +1,64 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_fail_one_skip_one_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_group(_Group,Config) ->
+ Config.
+
+end_per_group(_Group,_Config) ->
+ ok.
+
+init_per_testcase(test_case2, Config) ->
+ {skip,"skip it"};
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+groups() ->
+ [{group1,[parallel],[{group2,[parallel],[test_case1,test_case2,test_case3]}]}].
+
+all() ->
+ [{group,group1}].
+
+%% Test cases starts here.
+test_case1(Config) ->
+ ok = nok.
+
+test_case2(Config) ->
+ ok.
+
+test_case3(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_per_suite_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_per_suite_SUITE.erl new file mode 100644 index 0000000000..136a15ec96 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_per_suite_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_fail_per_suite_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ ok = nok.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_exit_in_init_scope_suite_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_exit_in_init_scope_suite_cth_SUITE.erl new file mode 100644 index 0000000000..42be0a659e --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_exit_in_init_scope_suite_cth_SUITE.erl @@ -0,0 +1,50 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_exit_in_init_scope_suite_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[crash_init_cth]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_cth_SUITE.erl new file mode 100644 index 0000000000..628bca774c --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_cth_SUITE.erl @@ -0,0 +1,56 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_group_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(GroupName, Config) ->
+ [{ct_hooks,[empty_cth]}|Config].
+
+end_per_group(GroupName, Config) ->
+ ok.
+
+all() ->
+ [{group,group1}].
+
+groups() ->
+ [{group1,[],[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_state_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_state_cth_SUITE.erl new file mode 100644 index 0000000000..14ea52bf8c --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_state_cth_SUITE.erl @@ -0,0 +1,56 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_group_state_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(_GroupName, Config) ->
+ [{ct_hooks,[{empty_cth,[test]}]}|Config].
+
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+all() ->
+ [{group,group1}].
+
+groups() ->
+ [{group1,[],[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_cth_SUITE.erl new file mode 100644 index 0000000000..5c1658be44 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_cth_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_suite_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ [{ct_hooks,[empty_cth]}|Config].
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_state_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_state_cth_SUITE.erl new file mode 100644 index 0000000000..96d00e3b28 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_state_cth_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_suite_state_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ [{ct_hooks,[{empty_cth,[test]}]}|Config].
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl new file mode 100644 index 0000000000..fa632444c5 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl @@ -0,0 +1,110 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_tc_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+%%--------------------------------------------------------------------
+%% @doc
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Initiation before the whole suite
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%%
+%% @spec init_per_suite(Config) -> Config
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Cleanup after the whole suite
+%%
+%% @spec end_per_suite(Config) -> _
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Case - atom()
+%% Name of the test case that is about to be run.
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Initiation before each test case
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%% Initiation before each test case
+%%
+%% @spec init_per_testcase(TestCase, Config) -> Config
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ [{ct_hooks,[empty_cth]}|Config].
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Case - atom()
+%% Name of the test case that is about to be run.
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Cleanup after each test case
+%%
+%% @spec end_per_testcase(TestCase, Config) -> _
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @doc
+%% TestCases - [Case]
+%% Case - atom()
+%% Name of a test case.
+%%
+%% Returns a list of all test cases in this test suite
+%%
+%% @spec all() -> TestCases
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+%%--------------------------------------------------------------------
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_cth_SUITE.erl new file mode 100644 index 0000000000..988a0969ca --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_cth_SUITE.erl @@ -0,0 +1,50 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_suite_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[empty_cth]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_state_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_state_cth_SUITE.erl new file mode 100644 index 0000000000..18b68fbcdc --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_state_cth_SUITE.erl @@ -0,0 +1,50 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_suite_state_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[{empty_cth,[test]}]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl new file mode 100644 index 0000000000..57fea347f6 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl @@ -0,0 +1,56 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_update_config_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ [{init_per_suite,now()}|Config].
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ [{init_per_testcase,now()}|Config].
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(GroupName, Config) ->
+ [{init_per_group,now()}|Config].
+
+end_per_group(GroupName, Config) ->
+ ok.
+
+all() ->
+ [{group,group1}].
+
+groups() ->
+ [{group1,[],[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl new file mode 100644 index 0000000000..5d07cd3dea --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -0,0 +1,278 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% @doc Common Test Example Suite Callback module.
+%%%
+%%% <p>This module gives an example of a common test CTH (Common Test Hook).
+%%% There are many ways to add a CTH to a test run, you can do it either in
+%%% the command line using -ct_hook, in a test spec using
+%%% {ct_hook,M} or in the suite it self by returning ct_hook
+%%% from either suite/0, init_per_suite/1, init_per_group/2 and
+%%% init_per_testcase/2. The scope of the CTH is determined by where is it
+%%% started. If it is started in the command line or test spec then it will
+%%% be stopped at the end of all tests. If it is started in init_per_suite,
+%%% it will be stopped after end_per_suite and so on. See terminate
+%%% documentation for a table describing the scoping machanics.
+%%%
+%%% All of callbacks except init/1 in a CTH are optional.</p>
+
+-module(empty_cth).
+
+%% CT Hooks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-type proplist() :: list({atom(),term()}).
+-type config() :: proplist().
+-type reason() :: term().
+-type skip_or_fail() :: {skip, reason()} |
+ {auto_skip, reason()} |
+ {fail, reason()} |
+ {'EXIT',reason()}.
+
+-record(state, { id = ?MODULE :: term()}).
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. It should return an state for this CTH.
+-spec init(Id :: term(), Opts :: proplist()) ->
+ State :: #state{}.
+init(Id, Opts) ->
+ gen_event:notify(?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, init, [Id, Opts]}}),
+ Opts.
+
+%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's
+%% return the same ID the seconds CTH is ignored. This function should NOT
+%% have any side effects as it might be called multiple times by common test.
+-spec id(Opts :: proplist()) ->
+ Id :: term().
+id(Opts) ->
+ gen_event:notify(?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, id, [Opts]}}),
+ now().
+
+%% @doc Called before init_per_suite is called. Note that this callback is
+%% only called if the CTH is added before init_per_suite is run (eg. in a test
+%% specification, suite/0 function etc).
+%% You can change the config in the this function.
+-spec pre_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_suite(Suite,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_init_per_suite,
+ [Suite,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after init_per_suite.
+%% you can change the return value in this function.
+-spec post_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_suite(Suite,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_init_per_suite,
+ [Suite,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called before end_per_suite. The config/state can be changed here,
+%% though it will only affect the *end_per_suite function.
+-spec pre_end_per_suite(Suite :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_suite(Suite,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_end_per_suite,
+ [Suite,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after end_per_suite. Note that the config cannot be
+%% changed here, only the status of the suite.
+-spec post_end_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_suite(Suite,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_suite,
+ [Suite,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called before each init_per_group.
+%% You can change the config in this function.
+-spec pre_init_per_group(Group :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_group(Group,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_init_per_group,
+ [Group,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after each init_per_group.
+%% You can change the return value in this function.
+-spec post_init_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_group(Group,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_init_per_group,
+ [Group,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called after each end_per_group. The config/state can be changed here,
+%% though it will only affect the *end_per_group functions.
+-spec pre_end_per_group(Group :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_group(Group,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_end_per_group,
+ [Group,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after each end_per_group. Note that the config cannot be
+%% changed here, only the status of the group.
+-spec post_end_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_group(Group,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_group,
+ [Group,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called before each test case.
+%% You can change the config in this function.
+-spec pre_init_per_testcase(TC :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_testcase(TC,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_init_per_testcase,
+ [TC,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after each test case. Note that the config cannot be
+%% changed here, only the status of the test case.
+-spec post_end_per_testcase(TC :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_testcase(TC,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_testcase,
+ [TC,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_tc if the suite, group or test case failed.
+%% This function should be used for extra cleanup which might be needed.
+%% It is not possible to modify the config or the status of the test run.
+-spec on_tc_fail(TC :: init_per_suite | end_per_suite |
+ init_per_group | end_per_group | atom(),
+ Reason :: term(), State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_fail(TC, Reason, State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, on_tc_fail,
+ [TC,Reason,State]}}),
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. Test case can be
+%% end_per_suite, init_per_group, end_per_group and the actual test cases.
+-spec on_tc_skip(TC :: end_per_suite |
+ init_per_group | end_per_group | atom(),
+ {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |
+ {tc_user_skip, {skipped, Reason :: term()}},
+ State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_skip(TC, Reason, State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, on_tc_skip,
+ [TC,Reason,State]}}),
+ State.
+
+%% @doc Called when the scope of the CTH is done, this depends on
+%% when the CTH was specified. This translation table describes when this
+%% function is called.
+%%
+%% | Started in | terminate called |
+%% |---------------------|-------------------------|
+%% | command_line | after all tests are run |
+%% | test spec | after all tests are run |
+%% | suite/0 | after SUITE is done |
+%% | init_per_suite/1 | after SUITE is done |
+%% | init_per_group/2 | after group is done |
+%% |-----------------------------------------------|
+%%
+-spec terminate(State :: #state{}) ->
+ term().
+terminate(State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, terminate, [State]}}),
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl new file mode 100644 index 0000000000..b4c26259a6 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl @@ -0,0 +1,72 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fail_post_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {{fail, "Test failure"}, State}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl new file mode 100644 index 0000000000..acf80a1b2e --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl @@ -0,0 +1,72 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fail_pre_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {{fail, "Test failure"}, State}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/id_no_init_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/id_no_init_cth.erl new file mode 100644 index 0000000000..58ed400e1c --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/id_no_init_cth.erl @@ -0,0 +1,32 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(id_no_init_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([id/1]).
+
+id(Opts) ->
+ empty_cth:id(Opts).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_cth.erl new file mode 100644 index 0000000000..a18f4bf2f3 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_cth.erl @@ -0,0 +1,33 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(minimal_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([init/2]).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl new file mode 100644 index 0000000000..79cd55f68e --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl @@ -0,0 +1,38 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(minimal_terminate_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([init/2]).
+-export([terminate/1]).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+terminate(State) ->
+ empty_cth:terminate(State).
+
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl new file mode 100644 index 0000000000..01a932bd59 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl @@ -0,0 +1,74 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(recover_post_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,{'EXIT',Reason} = Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {lists:keydelete(tc_status,1,Config),State};
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl new file mode 100644 index 0000000000..acfb93fe26 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl @@ -0,0 +1,75 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(same_id_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts),
+ ?MODULE.
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl new file mode 100644 index 0000000000..6d4605b33b --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl @@ -0,0 +1,72 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_post_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {{skip, "Test skip"}, State}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl new file mode 100644 index 0000000000..49efd0d0cd --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl @@ -0,0 +1,73 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_pre_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {{skip, "Test skip"}, State}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl new file mode 100644 index 0000000000..53d75e6ce3 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl @@ -0,0 +1,83 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(state_update_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ State = empty_cth:init(Id, Opts),
+ [init|State].
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {Config, [pre_init_per_suite|State]}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {Config, [post_init_per_suite|State]}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State),
+ {Config, [pre_end_per_suite|State]}.
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State),
+ {Return, [post_end_per_suite|State]}.
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State),
+ {Config, [pre_init_per_group|State]}.
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State),
+ {Return, [post_init_per_group|State]}.
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State),
+ {Config, [pre_end_per_group|State]}.
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State),
+ {Return, [post_end_per_group|State]}.
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State),
+ {Config, [pre_init_per_testcase|State]}.
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State),
+ {Return, [post_end_per_testcase|State]}.
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State),
+ [on_tc_fail|State].
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State),
+ [on_tc_skip|State].
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl new file mode 100644 index 0000000000..4c44ef025b --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl @@ -0,0 +1,71 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(undef_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(_Suite, _Config, _State) ->
+ lists:flaten([1,2,[3,4]]).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl new file mode 100644 index 0000000000..788ef2cec2 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl @@ -0,0 +1,82 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(update_config_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {[{pre_init_per_suite,now()}|Config],State}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {[{post_init_per_suite,now()}|Return],State}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State),
+ {[{pre_end_per_suite,now()}|Config],State}.
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State),
+ NewConfig = [{post_end_per_suite,now()}|Config],
+ {NewConfig,NewConfig}.
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State),
+ {[{pre_init_per_group,now()}|Config],State}.
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State),
+ {[{post_init_per_group,now()}|Return],State}.
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State),
+ {[{pre_end_per_group,now()}|Config],State}.
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State),
+ {[{post_end_per_group,now()}|Config],State}.
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State),
+ {[{pre_init_per_testcase,now()}|Config],State}.
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State),
+ {[{post_end_per_testcase,now()}|Config],State}.
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 5e9792f02c..b4f1a0e71f 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -58,6 +58,10 @@ init_per_suite(Config, Level) -> _ -> ok end, + + start_slave(Config, Level). + +start_slave(Config,Level) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), test_server:format(0, "Trying to start ~s~n", ["ct@"++Host]), @@ -136,9 +140,16 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, Config) -> CTNode = ?config(ct_node, Config), - wait_for_ct_stop(CTNode), - ok. - + case wait_for_ct_stop(CTNode) of + %% Common test was not stopped to we restart node. + false -> + cover:stop(CTNode), + slave:stop(CTNode), + start_slave(Config,proplists:get_value(trace_level,Config)), + {fail, "Could not stop common_test"}; + true -> + ok + end. %%%----------------------------------------------------------------- %%% @@ -229,11 +240,11 @@ wait_for_ct_stop(CTNode) -> wait_for_ct_stop(0, CTNode) -> test_server:format(0, "Giving up! Stopping ~p.", [CTNode]), - ok; + false; wait_for_ct_stop(Retries, CTNode) -> case rpc:call(CTNode, erlang, whereis, [ct_util_server]) of undefined -> - ok; + true; Pid -> test_server:format(0, "Waiting for CT (~p) to finish (~p)...", [Pid,Retries]), @@ -876,22 +887,49 @@ locate({TEH,tc_done,{undefined,undefined,{testcase_aborted, nomatch end; -%% matches any event of type Name -locate({TEH,Name,Data}, Node, [Ev|Evs], Config) when Data == '_' -> - case Ev of - {TEH,#event{name=Name, node=Node}} -> - {Config,Evs}; +%% Negative matching: Given two events, the first should not be present before +%% the other is matched. +locate({negative,NotMatch, Match} = Neg, Node, Evs, Config) -> + case locate(NotMatch, Node, Evs, Config) of + nomatch -> + locate(Match, Node, Evs, Config); _ -> - nomatch + exit({found_negative_event,Neg}) end; -locate({TEH,Name,Data}, Node, [Ev|Evs], Config) -> - case Ev of - {TEH,#event{name=Name, node=Node, data=Data}} -> - {Config,Evs}; - _ -> +%% matches any event of type Name +locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name, + data = EvData, + node = Node}}|Evs], + Config) -> + try match_data(Data, EvData) of + match -> + {Config,Evs} + catch _:_ -> nomatch - end. + end; + +locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) -> + nomatch. + +match_data(D,D) -> + match; +match_data('_',_) -> + match; +match_data(Fun,Data) when is_function(Fun) -> + Fun(Data); +match_data('$proplist',Proplist) -> + match_data( + fun(List) -> + lists:foreach(fun({_,_}) -> ok end,List) + end,Proplist); +match_data([H1|MatchT],[H2|ValT]) -> + match_data(H1,H2), + match_data(MatchT,ValT); +match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> + match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); +match_data([],[]) -> + match. log_events(TC, Events, PrivDir) -> LogFile = filename:join(PrivDir, atom_to_list(TC)++".events"), diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index ee121e5bb6..13ff02cc6c 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -935,8 +935,7 @@ spawn_fw_call(Mod,{init_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why, Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, %% if init_per_testcase fails, the test case %% should be skipped - case catch test_server_sup:framework_call( - end_tc,[?pl2a(Mod),Func,{Pid,Skip,[[]]}]) of + case catch do_end_tc_call(Mod,Func,{Pid,Skip,[[]]},Why) of {'EXIT',FwEndTCErr} -> exit({fw_notify_done,end_tc,FwEndTCErr}); _ -> @@ -955,11 +954,9 @@ spawn_fw_call(Mod,{end_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why, Conf = [{tc_status,ok}], %% if end_per_testcase fails, the test case should be %% reported successful with a warning printed as comment - case catch test_server_sup:framework_call(end_tc, - [?pl2a(Mod),Func, - {Pid, - {failed,{Mod,end_per_testcase,Why}}, - [Conf]}]) of + case catch do_end_tc_call(Mod,Func,{Pid, + {failed,{Mod,end_per_testcase,Why}}, + [Conf]}, Why) of {'EXIT',FwEndTCErr} -> exit({fw_notify_done,end_tc,FwEndTCErr}); _ -> @@ -1001,9 +998,7 @@ spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) -> ok end, Conf = [{tc_status,{failed,timetrap_timeout}}], - case catch test_server_sup:framework_call(end_tc, - [?pl2a(Mod),Func, - {Pid,Error,[Conf]}]) of + case catch do_end_tc_call(Mod,Func,{Pid,Error,[Conf]},Error) of {'EXIT',FwEndTCErr} -> exit({fw_notify_done,end_tc,FwEndTCErr}); _ -> @@ -1069,27 +1064,27 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, {{Time,Value},Loc,Opts} = case test_server_sup:framework_call(init_tc,[?pl2a(Mod),Func,Args0], - {ok,Args0}) of + {ok, Args0}) of {ok,Args} -> run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback); Error = {error,_Reason} -> - test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Error,Args0}]), - {{0,{skip,{failed,Error}}},{Mod,Func},[]}; + NewResult = do_end_tc_call(Mod,Func,{Error,Args0}, + {skip,{failed,Error}}), + {{0,NewResult},{Mod,Func},[]}; {fail,Reason} -> [Conf] = Args0, Conf1 = [{tc_status,{failed,Reason}} | Conf], fw_error_notify(Mod, Func, Conf, Reason), - test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func, - {{error,Reason},[Conf1]}]), - {{0,{failed,Reason}},{Mod,Func},[]}; + NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf1]}, + {fail, Reason}), + {{0,NewResult},{Mod,Func},[]}; Skip = {skip,_Reason} -> - test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Skip,Args0}]), - {{0,Skip},{Mod,Func},[]}; + NewResult = do_end_tc_call(Mod,Func,{Skip,Args0},Skip), + {{0,NewResult},{Mod,Func},[]}; {auto_skip,Reason} -> - test_server_sup:framework_call(end_tc,[?pl2a(Mod), - Func, - {{skip,Reason},Args0}]), - {{0,{skip,{fw_auto_skip,Reason}}},{Mod,Func},[]} + NewResult = do_end_tc_call(Mod, Func, {{skip,Reason},Args0}, + {skip, {fw_auto_skip,Reason}}), + {{0,NewResult},{Mod,Func},[]} end, exit({Ref,Time,Value,Loc,Opts}). @@ -1103,14 +1098,14 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> Skip = {skip,Reason} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}}], - test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Skip,[Conf]}]), - {{0,{skip,Reason}},Line,[]}; + NewRes = do_end_tc_call(Mod,Func,{Skip,[Conf]}, Skip), + {{0,NewRes},Line,[]}; {skip_and_save,Reason,SaveCfg} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}], - test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func, - {{skip,Reason},[Conf]}]), - {{0,{skip,Reason}},Line,[]}; + NewRes = do_end_tc_call(Mod, Func, {{skip, Reason}, [Conf]}, + {skip, Reason}), + {{0,NewRes},Line,[]}; {ok,NewConf} -> put(test_server_init_or_end_conf,undefined), %% call user callback function if defined @@ -1155,13 +1150,12 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> {FWReturn,TSReturn,EndConf1} end, put(test_server_init_or_end_conf,undefined), - case test_server_sup:framework_call(end_tc, [?pl2a(Mod), Func, - {FWReturn1,[EndConf2]}]) of - {fail,Reason} -> - fw_error_notify(Mod, Func, EndConf2, Reason), - {{T,{failed,Reason}},{Mod,Func},[]}; - _ -> - {{T,TSReturn1},Loc,[]} + case do_end_tc_call(Mod, Func, {FWReturn1,[EndConf2]}, TSReturn1) of + {failed,Reason} = NewReturn -> + fw_error_notify(Mod,Func,EndConf2, Reason), + {{T,NewReturn},{Mod,Func},[]}; + NewReturn -> + {{T,NewReturn},Loc,[]} end end; skip_init -> @@ -1179,10 +1173,36 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> {{T,Return},Loc} = {ts_tc(Mod, Func, Args2),get_loc()}, %% call user callback function if defined Return1 = user_callback(TCCallback, Mod, Func, 'end', Return), - {Return2,Opts} = process_return_val([Return1], Mod,Func,Args1, Loc, Return1), + {Return2,Opts} = process_return_val([Return1], Mod, Func, + Args1, Loc, Return1), {{T,Return2},Loc,Opts} end. +do_end_tc_call(M,F,Res,Return) -> + Ref = make_ref(), + case test_server_sup:framework_call( + end_tc, [?pl2a(M),F,Res], Ref) of + {fail,FWReason} -> + {failed,FWReason}; + Ref -> + case test_server_sup:framework_call( + end_tc, [?pl2a(M),F,Res, Return], ok) of + {fail,FWReason} -> + {failed,FWReason}; + ok -> + case Return of + {fail,Reason} -> + {failed,Reason}; + Return -> + Return + end; + NewReturn -> + NewReturn + end; + _ -> + Return + end. + %% the return value is a list and we have to check if it contains %% the result of an end conf case or if it's a Config list process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) -> @@ -1197,13 +1217,13 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) -> end, Return) of true -> % must be return value from end conf case process_return_val1(Return, M,F,A, Loc, Final, []); - false -> % must be Config value from init conf case - case test_server_sup:framework_call(end_tc, [?pl2a(M),F,{ok,A}]) of - {fail,FWReason} -> + false -> % must be Config value from init conf case + case do_end_tc_call(M,F,{ok,A}, Return) of + {failed, FWReason} = Failed -> fw_error_notify(M,F,A, FWReason), - {{failed,FWReason},[]}; - _ -> - {Return,[]} + {Failed, []}; + NewReturn -> + {NewReturn, []} end end; %% the return value is not a list, so it's the return value from an @@ -1211,16 +1231,16 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) -> process_return_val(Return, M,F,A, Loc, Final) -> process_return_val1(Return, M,F,A, Loc, Final, []). -process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts) when E=='EXIT'; - E==failed -> +process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts) + when E=='EXIT'; + E==failed -> fw_error_notify(M,F,A, TCError, mod_loc(Loc)), - case test_server_sup:framework_call(end_tc, - [?pl2a(M),F,{{error,TCError}, - [[{tc_status,{failed,TCError}}|Args]]}]) of - {fail,FWReason} -> + case do_end_tc_call(M,F,{{error,TCError}, + [[{tc_status,{failed,TCError}}|Args]]}, Failed) of + {failed,FWReason} -> {{failed,FWReason},SaveOpts}; - _ -> - {Failed,SaveOpts} + NewReturn -> + {NewReturn,SaveOpts} end; process_return_val1([SaveCfg={save_config,_}|Opts], M,F,[Args], Loc, Final, SaveOpts) -> process_return_val1(Opts, M,F,[[SaveCfg|Args]], Loc, Final, SaveOpts); @@ -1234,11 +1254,11 @@ process_return_val1([RetVal={Tag,_}|Opts], M,F,A, Loc, _, SaveOpts) when Tag==sk process_return_val1([_|Opts], M,F,A, Loc, Final, SaveOpts) -> process_return_val1(Opts, M,F,A, Loc, Final, SaveOpts); process_return_val1([], M,F,A, _Loc, Final, SaveOpts) -> - case test_server_sup:framework_call(end_tc, [?pl2a(M),F,{Final,A}]) of - {fail,FWReason} -> + case do_end_tc_call(M,F,{Final,A}, Final) of + {failed,FWReason} -> {{failed,FWReason},SaveOpts}; - _ -> - {Final,lists:reverse(SaveOpts)} + NewReturn -> + {NewReturn,lists:reverse(SaveOpts)} end. user_callback(undefined, _, _, _, Args) -> @@ -1263,7 +1283,7 @@ init_per_testcase(Mod, Func, Args) -> false -> code:load_file(Mod); _ -> ok end, - %% init_per_testcase defined, returns new configuration +%% init_per_testcase defined, returns new configuration case erlang:function_exported(Mod,init_per_testcase,2) of true -> case catch my_apply(Mod, init_per_testcase, [Func|Args]) of @@ -1306,8 +1326,8 @@ init_per_testcase(Mod, Func, Args) -> {skip,{failed,{Mod,init_per_testcase,Other}}} end; false -> - %% Optional init_per_testcase not defined - %% keep quiet. +%% Optional init_per_testcase not defined +%% keep quiet. [Config] = Args, {ok, Config} end. diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 1dc5646184..977a2b1785 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -3552,7 +3552,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Where, {_,{'EXIT',Reason}} -> progress(failed, Num, Mod, Func, Loc, Reason, Time, Comment, Style); - {_, {failed, Reason}} -> + {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed -> progress(failed, Num, Mod, Func, Loc, Reason, Time, Comment, Style); {_, {Skip, Reason}} when Skip==skip; Skip==skipped -> |