diff options
Diffstat (limited to 'lib/common_test')
74 files changed, 6318 insertions, 518 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index 6322860088..3ea6ae65d5 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# Copyright Ericsson AB 2003-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -51,8 +51,8 @@ CT_MODULES = \ CT_XML_FILES = $(CT_MODULES:=.xml) XML_APPLICATION_FILES = ref_man.xml -XML_REF1_FILES = run_test.xml -XML_REF3_FILES = $(CT_XML_FILES) +XML_REF1_FILES = ct_run.xml +XML_REF3_FILES = $(CT_XML_FILES) ct_hooks.xml XML_REF6_FILES = common_test_app.xml XML_PART_FILES = part.xml @@ -71,6 +71,7 @@ XML_CHAPTER_FILES = \ cover_chapter.xml \ ct_master_chapter.xml \ event_handler_chapter.xml \ + ct_hooks_chapter.xml \ dependencies_chapter.xml \ notes.xml \ notes_history.xml diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index e30eef2488..1ee73b890b 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2003</year><year>2010</year> + <year>2003</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -131,7 +131,8 @@ <type> <v> Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | - {silent_connections,Conns} | {stylesheet,CSSFile}</v> + {silent_connections,Conns} | {stylesheet,CSSFile} | + {ct_hooks, CTHs}</v> <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} | {hours,integer()}</v> <v> MilliSec = integer()</v> @@ -143,6 +144,9 @@ <v> UserData = term()</v> <v> Conns = [atom()]</v> <v> CSSFile = string()</v> + <v> CTHs = [CTHModule | {CTHModule, CTHInitArgs}]</v> + <v> CTHModule = atom()</v> + <v> CTHInitArgs = term()</v> </type> <desc> @@ -170,6 +174,10 @@ <p>With <c>userdata</c>, it is possible for the user to specify arbitrary test suite related information which can be read by calling <c>ct:userdata/2</c>.</p> + + <p>The <c>ct_hooks</c> tag specifies which + <seealso marker="ct_hooks_chapter">Common Test Hooks</seealso> + are to be run together with this suite.</p> <p>Other tuples than the ones defined will simply be ignored.</p> diff --git a/lib/common_test/doc/src/config_file_chapter.xml b/lib/common_test/doc/src/config_file_chapter.xml index 77b0c0c0b7..59151a73ec 100644 --- a/lib/common_test/doc/src/config_file_chapter.xml +++ b/lib/common_test/doc/src/config_file_chapter.xml @@ -248,7 +248,7 @@ <p><c>Callback:check_parameter/1</c></p> <p>The input argument will be passed from Common Test, as defined in the test - specification or given as an option to <c>run_test</c>.</p> + specification or given as an option to <c>ct_run</c> or <c>ct:run_test</c>.</p> <p>The return value should be any of the following values indicating if given configuration parameter is valid:</p> diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index 6e4f59ef73..b7162cb542 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2009</year> + <year>2006</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -94,10 +94,10 @@ <p>To activate the code coverage support, you simply specify the name of the cover specification file as you start Common Test. - This you do either by using the <c>-cover</c> flag with <c>run_test</c>. + This you do either by using the <c>-cover</c> flag with <c>ct_run</c>. Example:</p> - <p><c>$ run_test -dir $TESTOBJS/db -cover $TESTOBJS/db/config/db.coverspec</c></p> + <p><c>$ ct_run -dir $TESTOBJS/db -cover $TESTOBJS/db/config/db.coverspec</c></p> <p>You may also pass the cover specification file name in a call to <c>ct:run_test/1</c>, by adding a <c>{cover,CoverSpec}</c> diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml new file mode 100644 index 0000000000..7d5c9f4750 --- /dev/null +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -0,0 +1,556 @@ +<?xml version="1.0" encoding="UTF-8" ?> + +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2010</year><year>2011</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Common Test Hooks</title> + <prepared>Lukas Larsson</prepared> + <responsible>Lukas Larsson</responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date>2010-12-02</date> + <rev>PA1</rev> + <file>ct_hooks.sgml</file> + </header> + <module>ct_hooks</module> + <modulesummary>A callback interface on top of Common Test</modulesummary> + + <description> + + <warning><p>This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.</p></warning> + + <p>The <em>Common Test Hook</em> (henceforth called CTH) framework allows + extensions of the default behaviour of Common Test by means of callbacks + before and after all test suite calls. It is meant for advanced users of + Common Test which want to abstract out behaviour which is common to + multiple test suites. </p> + + <p>In brief, Common Test Hooks allows you to:</p> + + <list> + <item>Manipulate the runtime config before each suite + configuration call</item> + <item>Manipulate the return of all suite configuration calls and in + extension the result of the test themselves.</item> + </list> + + <p>The following sections describe the mandatory and optional CTH + functions Common Test will call during test execution. For more details + see <seealso marker="ct_hooks_chapter">Common Test Hooks</seealso> in + the User's Guide.</p> + + <p>For information about how to add a CTH to your suite see + <seealso marker="ct_hooks_chapter#installing">Installing a CTH + </seealso> in the User's Guide.</p> + + <note><p>See the + <seealso marker="ct_hooks_chapter#example">Example CTH</seealso> + in the User's Guide for a minimal example of a CTH. </p></note> + + </description> + + <section> + <title>CALLBACK FUNCTIONS</title> + <p>The following functions define the callback interface + for a Common Test Hook.</p> + </section> + + <funcs> + <func> + <name>Module:init(Id, Opts) -> State</name> + <fsummary>Initiates the Common Test Hook</fsummary> + <type> + <v>Id = reference() | term()</v> + <v>Opts = term()</v> + <v>State = term()</v> + </type> + + <desc> + <p> MANDATORY </p> + + <p>Always called before any other callback function. + Use this to initiate any common state. + It should return a state for this CTH.</p> + + <p><c>Id</c> is the return value of + <seealso marker="#Module:id-1">id/1</seealso>, or a <c>reference</c> + (created using + <seealso marker="erts:erlang#make_ref-0">make_ref/0</seealso>) + if <seealso marker="#Module:id-1">id/1</seealso> is not implemented. + </p> + + <p>For details about when init is called see + <seealso marker="ct_hooks_chapter#scope">scope</seealso> + in the User's Guide.</p> + + </desc> + </func> + + <func> + <name>Module:pre_init_per_suite(SuiteName, Config, CTHState) -> + Result</name> + <fsummary>Called before init_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {Return, NewCTHState}</v> + <v>Return = NewConfig | SkipOrFail</v> + <v>SkipOrFail = {fail, Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> if it exists. + It typically contains initialization/logging which needs to be done + before init_per_suite is called. + If <c>{skip,Reason}</c> or <c>{fail,Reason}</c> is returned, + init_per_suite and all test cases of the suite will be skipped and + Reason printed in the overview log of the suite.</p> + + <p><c>SuiteName</c> is the name of the suite to be run.</p> + + <p><c>Config</c> is the original config list of the test suite.</p> + + <p><c>CTHState</c> is the current internal state of the CTH.</p> + + <p><c>Return</c> is the result of the init_per_suite function. + If it is <c>{skip,Reason}</c> or <c>{fail,Reason}</c> + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> will never be called, instead the initiation is considered + to be skipped/failed respectively. If a <c>NewConfig</c> list + is returned, <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> will be called with that <c>NewConfig</c> list. + See <seealso marker="ct_hooks_chapter#pre"> + Pre Hooks</seealso> in the User's Guide for more details.</p> + + + <p>Note that this function is only called if the CTH has been added + before init_per_suite is run, see + <seealso marker="ct_hooks_chapter#scope">CTH Scoping</seealso> + in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:post_init_per_suite(SuiteName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after init_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail, Reason} | {skip, Reason} | term()</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite</seealso> if it exists. It typically contains extra + checks to make sure that all the correct dependencies have + been started correctly.</p> + + <p><c>Return</c> is what + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> returned, i.e. {fail,Reason}, {skip,Reason}, a <c>Config</c> + list or a term describing how + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> failed.</p> + + <p><c>NewReturn</c> is the possibly modified return value of + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso>. It is here possible to recover from a failure in + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite + </seealso> by returning the <c>ConfigList</c> with the <c>tc_status</c> + element removed. See <seealso marker="ct_hooks_chapter#post"> + Post Hooks</seealso> in the User's Guide for more details.</p> + + <p><c>CTHState</c> is the current internal state of the CTH.</p> + + <p>Note that this function is only called if the CTH has been added + before or in init_per_suite, see + <seealso marker="ct_hooks_chapter#scope">CTH Scoping</seealso> + in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:pre_init_per_group(GroupName, Config, CTHState) -> + Result</name> + <fsummary>Called before init_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> instead.</p> + </desc> + </func> + + <func> + <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after init_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group</seealso> instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_init_per_testcase(TestcaseName, Config, CTHState) -> + Result</name> + <fsummary>Called before init_per_testcase</fsummary> + <type> + <v>TestcaseName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:init_per_testcase-2"> + init_per_testcase</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:init_per_testcase-2"> + init_per_testcase</seealso> function instead.</p> + + <p>Note that it is not possible to add CTH's here right now, + that feature might be added later, + but it would right now break backwards compatability.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) + -> Result</name> + <fsummary>Called after end_per_testcase</fsummary> + <type> + <v>TestcaseName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_testcase-2"> + end_per_testcase</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_testcase-2"> + end_per_testcase</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_end_per_group(GroupName, Config, CTHState) -> + Result</name> + <fsummary>Called before end_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after end_per_group</fsummary> + <type> + <v>GroupName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_group-2"> + end_per_group</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:pre_end_per_suite(SuiteName, Config, CTHState) -> + Result</name> + <fsummary>Called before end_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = NewConfig = [{Key,Value}]</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewConfig | SkipOrFail, NewCTHState}</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called before + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:post_end_per_suite(SuiteName, Config, Return, CTHState) -> + Result</name> + <fsummary>Called after end_per_suite</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Config = [{Key,Value}]</v> + <v>Return = NewReturn = Config | SkipOrFail | term()</v> + <v>SkipOrFail = {fail,Reason} | {skip, Reason}</v> + <v>CTHState = NewCTHState = term()</v> + <v>Result = {NewReturn, NewCTHState}</v> + <v>Key = atom()</v> + <v>Value = term()</v> + <v>Reason = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called after + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> if it exists. It behaves the same way as + <seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, but for the + <seealso marker="common_test#Module:end_per_suite-1"> + end_per_suite</seealso> function instead.</p> + </desc> + </func> + + <func> + <name>Module:on_tc_fail(TestcaseName, Reason, CTHState) -> + NewCTHState</name> + <fsummary>Called after the CTH scope ends</fsummary> + <type> + <v>TestcaseName = init_per_suite | end_per_suite | + init_per_group | end_per_group | atom()</v> + <v>Reason = term()</v> + <v>CTHState = NewCTHState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called whenever a testcase fails. + It is called after the post function has been called for + the testcase which failed. i.e. + if init_per_suite fails this function is called after + <seealso marker="#Module:post_init_per_suite-4"> + post_init_per_suite</seealso>, and if a testcase fails it is called + after <seealso marker="#Module:post_end_per_testcase-4"> + post_end_per_testcase</seealso>.</p> + + <p>The data which comes with the Reason follows the same format as the + <seealso marker="event_handler_chapter#failreason">FailReason + </seealso> in the <seealso marker="event_handler_chapter#tc_done">tc_done</seealso> event. + See <seealso marker="event_handler_chapter#events">Event Handling + </seealso> in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:on_tc_skip(TestcaseName, Reason, CTHState) -> + NewCTHState</name> + <fsummary>Called after the CTH scope ends</fsummary> + <type> + <v>TestcaseName = end_per_suite | init_per_group | + end_per_group | atom()</v> + <v>Reason = {tc_auto_skip | tc_user_skip, term()}</v> + <v>CTHState = NewCTHState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called whenever a testcase is skipped. + It is called after the post function has been called for the + testcase which was skipped. + i.e. if init_per_group is skipped this function is called after + <seealso marker="#Module:post_init_per_suite-4">post_init_per_group + </seealso>, and if a testcase is skipped it is called after + <seealso marker="#Module:post_end_per_testcase-4">post_end_per_testcase + </seealso>.</p> + + <p>The data which comes with the Reason follows the same format as + <seealso marker="event_handler_chapter#tc_auto_skip">tc_auto_skip + </seealso> and <seealso marker="event_handler_chapter#tc_user_skip"> + tc_user_skip</seealso> events. + See <seealso marker="event_handler_chapter#events">Event Handling + </seealso> in the User's Guide for details.</p> + </desc> + </func> + + <func> + <name>Module:terminate(CTHState)</name> + <fsummary>Called after the CTH scope ends</fsummary> + <type> + <v>CTHState = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>This function is called at the end of a CTH's + <seealso marker="ct_hooks_chapter#scope">scope</seealso>. + </p> + </desc> + </func> + + <func> + <name>Module:id(Opts) -> Id</name> + <fsummary>Called before the init function of a CTH</fsummary> + <type> + <v>Opts = term()</v> + <v>Id = term()</v> + </type> + + <desc> + <p> OPTIONAL </p> + + <p>The <c>Id</c> is used to uniquely identify a CTH instance, + if two CTH's return the same <c>Id</c> the second CTH is ignored + and subsequent calls to the CTH will only be made to the first + instance. For more information see + <seealso marker="ct_hooks_chapter#installing">Installing a CTH + </seealso> in the User's Guide. + </p> + + <p>This function should NOT have any side effects as it might + be called multiple times by Common Test.</p> + + <p>If not implemented the CTH will act as if this function returned a + call to <c>make_ref/0</c>.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml new file mode 100644 index 0000000000..fc5ab48e1b --- /dev/null +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -0,0 +1,401 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<!DOCTYPE chapter SYSTEM "chapter.dtd"> + +<chapter> + <header> + <copyright> + <year>2011</year><year>2011</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + The contents of this file are subject to the Erlang Public License, + Version 1.1, (the "License"); you may not use this file except in + compliance with the License. You should have received a copy of the + Erlang Public License along with this software. If not, it can be + retrieved online at http://www.erlang.org/. + + Software distributed under the License is distributed on an "AS IS" + basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + the License for the specific language governing rights and limitations + under the License. + + </legalnotice> + + <title>Common Test Hooks</title> + <prepared>Lukas Larsson</prepared> + <docno></docno> + <date></date> + <rev></rev> + <file>ct_hooks_chapter.xml</file> + </header> + + <marker id="general"></marker> + <section> + <title>General</title> + <warning><p>This feature is in alpha release right now. This means that the + interface may change in the future and that there may be bugs. We + encourage you to use this feature, but be prepared + that there might be bugs and that the interface might change + inbetween releases.</p></warning> + <p> + The <em>Common Test Hook</em> (henceforth called CTH) framework allows + extensions of the default behaviour of Common Test by means of hooks + before and after all test suite calls. CTHs allow advanced Common Test + users to abstract out behaviour which is common to multiple test suites + without littering all test suites with library calls. Some example + usages are: logging, starting and monitoring external systems, + building C files needed by the tests and much more!</p> + + <p>In brief, Common Test Hooks allows you to:</p> + + <list> + <item>Manipulate the runtime config before each suite + configuration call</item> + <item>Manipulate the return of all suite configuration calls and in + extension the result of the test themselves.</item> + </list> + + <p>The following sections describe how to use CTHs, when they are run + and how to manipulate your test results in a CTH</p> + + <warning><p>When executing within a CTH all timetraps are shutoff. So + if your CTH never returns, the entire test run will be stalled!</p> + </warning> + + </section> + + <marker id="installing"></marker> + <section> + <title>Installing a CTH</title> + <p>There are multiple ways to install a CTH in your test run. You can do it + for all tests in a run, for specific test suites and for specific groups + within a test suite. If you want a CTH to be present in all test suites + within your test run there are three different ways to accomplish that. + </p> + + <list> + <item>Add <c>-ct_hooks</c> as an argument to + <seealso marker="run_test_chapter#ct_run">ct_run</seealso>. + To add multiple CTHs using this method append them to each other + using the keyword <c>and</c>, i.e. + <c>ct_run -ct_hooks cth1 [{debug,true}] and cth2 ...</c>.</item> + <item>Add the <c>ct_hooks</c> tag to your + <seealso marker="run_test_chapter#test_specifications"> + Test Specification</seealso></item> + <item>Add the <c>ct_hooks</c> tag to your call to + <seealso marker="ct#run_test-1">ct:run_test/1</seealso></item> + </list> + + <p>You can also add CTHs within a test suite. This is done by returning + <c>{ct_hooks,[CTH]}</c> in the config list from + <seealso marker="common_test#Module:suite-0">suite/0</seealso>, + <seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite/1</seealso> or + <seealso marker="common_test#Module:init_per_group-2"> + init_per_group/2</seealso>. <c>CTH</c> in this case can be either + only the module name of the CTH or a tuple with the module name and the + initial arguments to the CTH. Eg: + <c>{ct_hooks,[my_cth_module]}</c> or + <c>{ct_hooks,[{my_cth_module,[{debug,true}]}]}</c></p> + + <section> + <title>Overriding CTHs</title> + <p>By default each installation of a CTH will cause a new instance of it + to be activated. This can cause problems if you want to be able to + override CTHs in test specifications while still having them in the + suite info function. The + <seealso marker="ct_hooks#Module:id-1">id/1</seealso> + callback exists to address this problem. By returning the same + <c>id</c> in both places, Common Test knows that this CTH + has already been installed and will not try to install it again.</p> + </section> + + </section> + + <marker id="scope"/> + <section> + <title>CTH Scope</title> + <p>Once the CTH is installed into a certain test run it will be there until + its scope is expired. The scope of a CTH depends on when it is + installed. + The <seealso marker="ct_hooks#Module:init-2">init/2</seealso> is + called at the beginning of the scope and the + <seealso marker="ct_hooks#Module:terminate-1">terminate/1 + </seealso> function is called when the scope ends.</p> + <table> + <row> + <cell><em>CTH Installed in</em></cell> + <cell><em>CTH scope begins before</em></cell> + <cell><em>CTH scope ends after</em></cell> + </row> + <row> + <cell><seealso marker="run_test_chapter#ct_run">ct_run</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="ct#run_test-1">ct:run_test</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="run_test_chapter#test_specifications"> + Test Specification</seealso></cell> + <cell>the first test suite is to be run.</cell> + <cell>the last test suite has been run.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:suite-0">suite/0 + </seealso></cell> + <cell><seealso marker="ct_hooks#Module:pre_init_per_suite-3"> + pre_init_per_suite/3</seealso> is called.</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> + post_end_per_suite/4</seealso> has been called for that test suite.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:init_per_suite-1"> + init_per_suite/1</seealso></cell> + <cell><seealso marker="ct_hooks#Module:post_init_per_suite-4"> + post_init_per_suite/4</seealso> is called.</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> + post_end_per_suite/4</seealso> has been called for that test suite.</cell> + </row> + <row> + <cell><seealso marker="common_test#Module:init_per_group-2"> + init_per_group/2</seealso></cell> + <cell><seealso marker="ct_hooks#Module:post_init_per_group-4"> + post_init_per_group/4</seealso> is called.</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> + post_end_per_group/4</seealso> has been called for that group.</cell> + </row> + <tcaption>Scope of a CTH</tcaption> + </table> + + <section> + <title>CTH Processes and Tables</title> + <p>CTHs are run with the same process scoping as normal test suites + i.e. a different process will execute the init_per_suite hooks then the + init_per_group or per_testcase hooks. So if you want to spawn a + process in the CTH you cannot link with the CTH process as it will exit + after the post hook ends. Also if you for some reason need an ETS + table with your CTH, you will have to spawn a process which handles + it.</p> + </section> + + </section> + + <marker id="manipulating"/> + <section> + <title>Manipulating tests</title> + <p>It is through CTHs possible to manipulate the results of tests and + configuration functions. The main purpose of doing this with CTHs is to + allow common patterns to be abstracted out from test test suites and applied to + multiple test suites without duplicating any code. All of the callback + functions for a CTH follow a common interface, this interface is + described below.</p> + + <p>It is only possible to hook into test function which exists in the test + suite. So in order for a CTH to hook in before + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + the <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso> + function must exist in the test suite.</p> + + <marker id="pre"/> + <section> + <title>Pre Hooks</title> + <p> + It is possible in a CTH to hook in behaviour before + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_testcase</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and + <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>. + This is done in the CTH functions called pre_<name of function>. + All of these functions take the same three arguments: <c>Name</c>, + <c>Config</c> and <c>CTHState</c>. The return value of the CTH function + is always a combination of an result for the suite/group/test and an + updated <c>CTHState</c>. If you want the test suite to continue on + executing you should return the config list which you want the test to + use as the result. If you for some reason want to skip/fail the test, + return a tuple with <c>skip</c> or <c>fail</c> and a reason as the + result. Example: + </p> + <code>pre_init_per_suite(SuiteName, Config, CTHState) -> + case db:connect() of + {error,_Reason} -> + {{fail, "Could not connect to DB"}, CTHState}; + {ok, Handle} -> + {[{db_handle, Handle} | Config], CTHState#state{ handle = Handle }} + end.</code> + + </section> + + <marker id="post"/> + <section> + <title>Post Hooks</title> + <p>It is also possible in a CTH to hook in behaviour after + <seealso marker="common_test#Module:init_per_suite-1">init_per_suite</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">init_per_group</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_testcase</seealso>, + <seealso marker="common_test#Module:init_per_suite-1">end_per_group</seealso> and + <seealso marker="common_test#Module:init_per_suite-1">end_per_suite</seealso>. + This is done in the CTH functions called post_<name of function>. + All of these function take the same four arguments: <c>Name</c>, + <c>Config</c>, <c>Return</c> and <c>CTHState</c>. <c>Config</c> in this + case is the same <c>Config</c> as the testcase is called with. + <c>Return</c> is the value returned by the testcase. If the testcase + failed by crashing, <c>Return</c> will be + <c>{'EXIT',{{Error,Reason},Stacktrace}}</c>.</p> + + <p>The return value of the CTH function is always a combination of an + result for the suite/group/test and an updated <c>CTHState</c>. If + you want the callback to not affect the outcome of the test you should + return the <c>Return</c> data as it is given to the CTH. You can also + modify the result of the test. By returning the <c>Config</c> list + with the <c>tc_status</c> element removed you can recover from a test + failure. As in all the pre hooks, it is also possible to fail/skip + the test case in the post hook. Example: </p> + + <code>post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + case db:check_consistency() of + true -> + %% DB is good, pass the test. + {proplists:delete(tc_status, Config), CTHState}; + false -> + %% DB is not good, mark as skipped instead of failing + {{skip, "DB is inconsisten!"}, CTHState} + end; +post_end_per_testcase(_TC, Config, Return, CTHState) -> + %% Do nothing if tc does not crash. + {Return, CTHState}.</code> + + <note>Recovering from a testcase failure using CTHs should only be done as + a last resort. If used wrongly it could become very difficult to + determine which tests pass or fail in a test run</note> + + </section> + + <marker id="skip_n_fail"/> + <section> + <title>Skip and Fail hooks</title> + <p> + After any post hook has been executed for all installed CTHs, + <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso> + or <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_skip</seealso> + might be called if the testcase failed or was skipped + respectively. You cannot affect the outcome of the tests any further at + this point. + </p> + </section> + + </section> + + <marker id="example"/> + <section> + <title>Example CTH</title> + <p>The CTH below will log information about a test run into a format + parseable by <seealso marker="kernel:file#consult-1">file:consult/1</seealso>. + </p> + <code>%%% @doc Common Test Example Common Test Hook module. +-module(example_cth). + +%% Callbacks +-export([id/1]). +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([post_init_per_suite/4]). +-export([pre_end_per_suite/3]). +-export([post_end_per_suite/4]). + +-export([pre_init_per_group/3]). +-export([post_init_per_group/4]). +-export([pre_end_per_group/3]). +-export([post_end_per_group/4]). + +-export([pre_init_per_testcase/3]). +-export([post_end_per_testcase/4]). + +-export([on_tc_fail/3]). +-export([on_tc_skip/3]). + +-export([terminate/1]). + +-record(state, { file_handle, total, suite_total, ts, tcs, data }). + +%% @doc Return a unique id for this CTH. +id(Opts) -> + proplists:get_value(filename, Opts, "/tmp/file.log"). + +%% @doc Always called before any other callback function. Use this to initiate +%% any common state. +init(Id, Opts) -> + {ok,D} = file:open(Id,[write]), + #state{ file_handle = D, total = 0, data = [] }. + +%% @doc Called before init_per_suite is called. +pre_init_per_suite(Suite,Config,State) -> + {Config, State#state{ suite_total = 0, tcs = [] }}. + +%% @doc Called after init_per_suite. +post_init_per_suite(Suite,Config,Return,State) -> + {Return, State}. + +%% @doc Called before end_per_suite. +pre_end_per_suite(Suite,Config,State) -> + {Config, State}. + +%% @doc Called after end_per_suite. +post_end_per_suite(Suite,Config,Return,State) -> + Data = {suites, Suite, State#state.suite_total, lists:reverse(State#state.tcs)}, + {Return, State#state{ data = [Data | State#state.data] , + total = State#state.total + State#state.suite_total } }. + +%% @doc Called before each init_per_group. +pre_init_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each init_per_group. +post_init_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called after each end_per_group. +pre_end_per_group(Group,Config,State) -> + {Config, State}. + +%% @doc Called after each end_per_group. +post_end_per_group(Group,Config,Return,State) -> + {Return, State}. + +%% @doc Called before each test case. +pre_init_per_testcase(TC,Config,State) -> + {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. + +%% @doc Called after each test case. +post_end_per_testcase(TC,Config,Return,State) -> + TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. + +%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. +on_tc_fail(TC, Reason, State) -> + State. + +%% @doc Called when a test case is skipped by either user action +%% or due to an init function failing. +on_tc_skip(TC, Reason, State) -> + State. + +%% @doc Called when the scope of the CTH is done +terminate(State) -> + io:format(State#state.file_handle, "~p.~n", + [{test_run, State#state.total, State#state.data}]), + file:close(State#state.file_handle), + ok.</code> + </section> + +</chapter> + + + + diff --git a/lib/common_test/doc/src/ct_master_chapter.xml b/lib/common_test/doc/src/ct_master_chapter.xml index 01f8e61d36..f4f0ecad62 100644 --- a/lib/common_test/doc/src/ct_master_chapter.xml +++ b/lib/common_test/doc/src/ct_master_chapter.xml @@ -188,7 +188,7 @@ <seealso marker="run_test_chapter#test_specifications">Running Test Suites</seealso> chapter). The result is that any test specified to run on a node with the same name as the Common Test node in question (typically <c>ct@somehost</c> if started - with the <c>run_test</c> program), will be performed. Tests without explicit + with the <c>ct_run</c> program), will be performed. Tests without explicit node association will always be performed too of course!</p> <note><p>It is recommended that absolute paths are used for log directories, diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/ct_run.xml index 2f0a94afba..1ab563d74f 100644 --- a/lib/common_test/doc/src/run_test.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -21,7 +21,7 @@ </legalnotice> - <title>The run_test program</title> + <title>The ct_run program</title> <prepared>Peter Andersson</prepared> <responsible>Peter Andersson</responsible> <docno></docno> @@ -29,18 +29,18 @@ <checked></checked> <date>2010-04-01</date> <rev>PA2</rev> - <file>run_test.xml</file> + <file>ct_run.xml</file> </header> - <com>run_test</com> + <com>ct_run</com> <comsummary>Program used for starting Common Test from the OS command line. </comsummary> <description> - <p>The <c>run_test</c> program is automatically installed with Erlang/OTP + <p>The <c>ct_run</c> program is automatically installed with Erlang/OTP and Common Test (please see the Installation chapter in the Common Test User's Guide for more information). The program accepts a number - of different start flags. Some flags trigger <c>run_test</c> + of different start flags. Some flags trigger <c>ct_run</c> to start the Common Test application and pass on data to it. Some flags start an Erlang node prepared for running Common Test in a particular mode.</p> @@ -50,20 +50,20 @@ shell (or an Erlang program). Please see the <c>ct</c> man page for details.</p> - <p><c>run_test</c> also accepts Erlang emulator flags. These are used - when <c>run_test</c> calls <c>erl</c> to start the Erlang node + <p><c>ct_run</c> also accepts Erlang emulator flags. These are used + when <c>ct_run</c> calls <c>erl</c> to start the Erlang node (making it possible to e.g. add directories to the code server path, change the cookie on the node, start additional applications, etc).</p> <p>With the optional flag:</p> <pre>-erl_args</pre> - <p>it's possible to divide the options on the <c>run_test</c> command line into + <p>it's possible to divide the options on the <c>ct_run</c> command line into two groups, one that Common Test should process (those preceding <c>-erl_args</c>), and one it should completely ignore and pass on directly to the emulator (those following <c>-erl_args</c>). Options preceding <c>-erl_args</c> that Common Test doesn't recognize, also get passed on to the emulator untouched. By means of <c>-erl_args</c> the user may specify flags with the same name, but - with different destinations, on the <c>run_test</c> command line.</p> + with different destinations, on the <c>ct_run</c> command line.</p> <p>If <c>-pa</c> or <c>-pz</c> flags are specified in the Common Test group of options (preceding <c>-erl_args</c>), relative directories will be converted to absolute and re-inserted into the code path by Common Test (to avoid @@ -72,17 +72,17 @@ following <c>-erl_args</c> on the command line. These directories are added to the code path normally (i.e. on specified form)</p> - <p>If <c>run_test</c> is called with option:</p> + <p>If <c>ct_run</c> is called with option:</p> <pre>-help</pre> <p>it prints all valid start flags to stdout.</p> </description> - <marker id="run_test"></marker> + <marker id="ct_run"></marker> <section> <title>Run tests from command line</title> <pre> - run_test [-dir TestDir1 TestDir2 .. TestDirN] | + ct_run [-dir TestDir1 TestDir2 .. TestDirN] | [-suite Suite1 Suite2 .. SuiteN [[-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]]] [-step [config | keep_inactive]] @@ -110,7 +110,7 @@ <section> <title>Run tests using test specification</title> <pre> - run_test -spec TestSpec1 TestSpec2 .. TestSpecN + ct_run -spec TestSpec1 TestSpec2 .. TestSpecN [-config ConfigFile1 ConfigFile2 .. ConfigFileN] [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 ConfigString2 and .. and CallbackModuleN ConfigStringN] @@ -136,7 +136,7 @@ <section> <title>Run tests in web based GUI</title> <pre> - run_test -vts [-browser Browser] + ct_run -vts [-browser Browser] [-dir TestDir1 TestDir2 .. TestDirN] | [-suite Suite [[-group Group] [-case Case]]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] @@ -152,12 +152,12 @@ <section> <title>Refresh the HTML index files</title> <pre> - run_test -refresh_logs [-logdir LogDir] [-basic_html]</pre> + ct_run -refresh_logs [-logdir LogDir] [-basic_html]</pre> </section> <section> <title>Run CT in interactive mode</title> <pre> - run_test -shell + ct_run -shell [-config ConfigFile1 ConfigFile2 ... ConfigFileN] [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 ConfigString2 and .. and CallbackModuleN ConfigStringN] @@ -166,7 +166,7 @@ <section> <title>Start a Common Test Master node</title> <pre> - run_test -ctmaster</pre> + ct_run -ctmaster</pre> </section> <section> diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index 7f5144b760..b41b233ce6 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2010</year> + <year>2006</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -61,14 +61,15 @@ itself.</p> </section> <section> + <marker id="usage"></marker> <title>Usage</title> <p>Event handlers may be installed by means of an <c>event_handler</c> - start flag (<c>run_test</c>) or option (<c>ct:run_test/1</c>), where the + start flag (<c>ct_run</c>) or option (<c>ct:run_test/1</c>), where the argument specifies the names of one or more event handler modules. Example:</p> - <p><c>$ run_test -suite test/my_SUITE -event_handler handlers/my_evh1 + <p><c>$ ct_run -suite test/my_SUITE -event_handler handlers/my_evh1 handlers/my_evh2 -pa $PWD/handlers</c></p> - <p>Use the <c><![CDATA[run_test -event_handler_init]]></c> option instead of + <p>Use the <c><![CDATA[ct_run -event_handler_init]]></c> option instead of <c><![CDATA[-event_handler]]></c> to pass start arguments to the event handler init function.</p> <p>All event handler modules must have gen_event behaviour. Note also that @@ -120,6 +121,7 @@ node the event has originated from (only relevant for CT Master event handlers). <c>data</c> is specific for the particular event.</p> + <marker id="events"></marker> <p><em>General events:</em></p> <list> @@ -172,6 +174,7 @@ are also given. </p></item> + <marker id="tc_done"/> <item><c>#event{name = tc_done, data = {Suite,FuncOrGroup,Result}}</c> <p><c>Suite = atom()</c>, name of the suite.</p> <p><c>FuncOrGroup = Func | {Conf,GroupName,GroupProperties}</c></p> @@ -181,12 +184,14 @@ (unknown if init- or end function times out).</p> <p><c>GroupProperties = list()</c>, list of execution properties for the group.</p> <p><c>Result = ok | {skipped,SkipReason} | {failed,FailReason}</c>, the result.</p> + <marker id="skipreason"/> <p><c>SkipReason = {require_failed,RequireInfo} | {require_failed_in_suite0,RequireInfo} | {failed,{Suite,init_per_testcase,FailInfo}} | UserTerm</c>, the reason why the case has been skipped.</p> - <p><c>FailReason = {error,FailInfo} | + <marker id="failreason"/> + <p><c>FailReason = {error,FailInfo} | {error,{RunTimeError,StackTrace}} | {timetrap_timeout,integer()} | {failed,{Suite,end_per_testcase,FailInfo}}</c>, reason for failure.</p> @@ -209,6 +214,7 @@ <c>end_per_testcase</c> for the case failed. </p></item> + <marker id="tc_auto_skip"></marker> <item><c>#event{name = tc_auto_skip, data = {Suite,Func,Reason}}</c> <p><c>Suite = atom()</c>, the name of the suite.</p> <p><c>Func = atom()</c>, the name of the test case or configuration function.</p> @@ -234,7 +240,8 @@ skipped because of <c>init_per_testcase</c> failing, since that information is carried with the <c>tc_done</c> event. </p></item> - + + <marker id="tc_user_skip"></marker> <item><c>#event{name = tc_user_skip, data = {Suite,TestCase,Comment}}</c> <p><c>Suite = atom()</c>, name of the suite.</p> <p><c>TestCase = atom()</c>, name of the test case.</p> diff --git a/lib/common_test/doc/src/install_chapter.xml b/lib/common_test/doc/src/install_chapter.xml index 828588a673..89c497962d 100644 --- a/lib/common_test/doc/src/install_chapter.xml +++ b/lib/common_test/doc/src/install_chapter.xml @@ -34,8 +34,8 @@ <title>General information</title> <p>The two main interfaces for running tests with Common Test - are an executable program named run_test and an - erlang module named <c>ct</c>. The run_test program + are an executable program named ct_run and an + erlang module named <c>ct</c>. The ct_run program is compiled for the underlying operating system (e.g. Unix/Linux or Windows) during the build of the Erlang/OTP system, and is installed automatically with other executable programs in @@ -43,22 +43,22 @@ The <c>ct</c> interface functions can be called from the Erlang shell, or from any Erlang function, on any supported platform.</p> - <p>A legacy Bourne shell script - also named run_test - exists, + <p>A legacy Bourne shell script - named run_test - exists, which may be manually generated and installed. This script may be used - instead of the run_test program mentioned above, e.g. if the user + instead of the ct_run program mentioned above, e.g. if the user wishes to modify or customize the Common Test start flags in a simpler - way than making changes to the run_test C program.</p> + way than making changes to the ct_run C program.</p> <p>The Common Test application is installed with the Erlang/OTP system and no additional installation step is required to start using - Common Test by means of the run_test executable program, and/or the interface + Common Test by means of the ct_run executable program, and/or the interface functions in the <c>ct</c> module. If you wish to use the legacy Bourne - shell script version of run_test, however, this script needs to be + shell script version run_test, however, this script needs to be generated first, according to the instructions below.</p> <p><note>Before reading on, please note that since Common Test version 1.5, the run_test shell script is no longer required for starting - tests with Common Test from the OS command line. The run_test + tests with Common Test from the OS command line. The ct_run program (descibed above) is the new recommended command line interface for Common Test. The shell script exists mainly for legacy reasons and may not be updated in future releases of Common Test. It may even be removed. diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index af9dbfa9ec..fef1222fcb 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -32,6 +32,95 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.5.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Added an option to test specs which allow the execution + of tests as is, instead of doing merging of tests on the + same "level". See the merge_tests directive the test + specification documentation.</p> + <p> + Own Id: OTP-9026 Aux Id: seq11768 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Alpha release of Common Test Hooks (CTH). CTHs allow the + users of common test to abtract out common behaviours + from test suites in a much more elegant and flexible way + than was possible before. Note that the addition of this + feature may introduce minor changes in the undocumented + behaviour of the interface inbetween common_test and + test_server.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8851</p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.5.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Updated ct:get_status documentation to describe + no_tests_running return value.</p> + <p> + Own Id: OTP-8895 Aux Id: seq11701 </p> + </item> + <item> + <p> + Fixed race condition test failures in the test suites + testing common test's parallel groups feature.</p> + <p> + Own Id: OTP-8921</p> + </item> + <item> + <p> + The include directive of testspecs now work when used on + a remote node.</p> + <p> + Own Id: OTP-8935 Aux Id: seq11731 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + ct:parse_table can now handle multiline sql rows</p> + <p> + Own Id: OTP-8907 Aux Id: seq11702 </p> + </item> + <item> + <p> + The run_test executable has been renamed to the less + generic ct_run to better work with other applications. + run_test will remain until R16B at which point it will be + removed.</p> + <p> + Own Id: OTP-8936</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.5.1</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index 53a4cb1bbf..3284bcadaa 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -4,7 +4,7 @@ <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -75,6 +75,7 @@ <xi:include href="ct_master_chapter.xml"/> <xi:include href="event_handler_chapter.xml"/> <xi:include href="dependencies_chapter.xml"/> + <xi:include href="ct_hooks_chapter.xml"/> <xi:include href="why_test_chapter.xml"/> </part> diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index 8be234d979..a9fdef7359 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2010</year> + <year>2003</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -63,7 +63,7 @@ Server application.</p> </description> <xi:include href="common_test_app.xml"/> - <xi:include href="run_test.xml"/> + <xi:include href="ct_run.xml"/> <!-- If you make modifications in the module list below, you also need to update CT_MODULES in Makefile. --> <xi:include href="ct.xml"/> @@ -76,6 +76,7 @@ <xi:include href="ct_telnet.xml"/> <xi:include href="unix_telnet.xml"/> <xi:include href="ct_slave.xml"/> + <xi:include href="ct_hooks.xml"/> </application> diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 1efff25f5b..e6fb85634f 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2010</year> + <year>2003</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -71,7 +71,7 @@ <p>If test suites or help modules include header files stored in other locations than the test directory, you may specify these include directories - by means of the <c><![CDATA[-include]]></c> flag with <c><![CDATA[run_test]]></c>, + by means of the <c><![CDATA[-include]]></c> flag with <c><![CDATA[ct_run]]></c>, or the <c><![CDATA[include]]></c> option with <c><![CDATA[ct:run_test/1]]></c>. In addition to this, an include path may be specified with an OS environment variable; <c><![CDATA[CT_INCLUDE_PATH]]></c>. Example (bash):</p> @@ -93,7 +93,7 @@ there instead.</p> <p>It is possible to disable the automatic compilation feature by using the - <c><![CDATA[-no_auto_compile]]></c> flag with <c><![CDATA[run_test]]></c>, or + <c><![CDATA[-no_auto_compile]]></c> flag with <c><![CDATA[ct_run]]></c>, or the <c><![CDATA[{auto_compile,false}]]></c> option with <c><![CDATA[ct:run_test/1]]></c>. With automatic compilation disabled, the user is responsible for compiling the test suite modules @@ -105,29 +105,30 @@ RPC from a remote node.</p> </section> + <marker id="ct_run"></marker> <section> <title>Running tests from the OS command line</title> - <p>The <c>run_test</c> program can be used for running tests from + <p>The <c>ct_run</c> program can be used for running tests from the OS command line, e.g. </p> <list> - <item><c><![CDATA[run_test -config <configfilenames> -dir <dirs>]]></c></item> - <item><c><![CDATA[run_test -config <configfilenames> -suite <suiteswithfullpath>]]></c> + <item><c><![CDATA[ct_run -config <configfilenames> -dir <dirs>]]></c></item> + <item><c><![CDATA[ct_run -config <configfilenames> -suite <suiteswithfullpath>]]></c> </item> - <item><c><![CDATA[run_test -userconfig <callbackmodulename> <configfilenames> -suite <suiteswithfullpath>]]></c> + <item><c><![CDATA[ct_run -userconfig <callbackmodulename> <configfilenames> -suite <suiteswithfullpath>]]></c> </item> - <item><c><![CDATA[run_test -config <configfilenames> -suite <suitewithfullpath> + <item><c><![CDATA[ct_run -config <configfilenames> -suite <suitewithfullpath> -group <groupnames> -case <casenames>]]></c></item> </list> <p>Examples:</p> - <p><c>$ run_test -config $CFGS/sys1.cfg $CFGS/sys2.cfg -dir $SYS1_TEST $SYS2_TEST</c></p> - <p><c>$ run_test -userconfig ct_config_xml $CFGS/sys1.xml $CFGS/sys2.xml -dir $SYS1_TEST $SYS2_TEST</c></p> - <p><c>$ run_test -suite $SYS1_TEST/setup_SUITE $SYS2_TEST/config_SUITE</c></p> - <p><c>$ run_test -suite $SYS1_TEST/setup_SUITE -case start stop</c></p> - <p><c>$ run_test -suite $SYS1_TEST/setup_SUITE -group installation -case start stop</c></p> + <p><c>$ ct_run -config $CFGS/sys1.cfg $CFGS/sys2.cfg -dir $SYS1_TEST $SYS2_TEST</c></p> + <p><c>$ ct_run -userconfig ct_config_xml $CFGS/sys1.xml $CFGS/sys2.xml -dir $SYS1_TEST $SYS2_TEST</c></p> + <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE $SYS2_TEST/config_SUITE</c></p> + <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE -case start stop</c></p> + <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE -group installation -case start stop</c></p> - <p>Other flags that may be used with <c>run_test</c>:</p> + <p>Other flags that may be used with <c>ct_run</c>:</p> <list> <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item> <item><c><![CDATA[-label <name_of_test_run>]]></c>, associates the test run with a name that gets printed @@ -147,6 +148,8 @@ <seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item> <item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install <seealso marker="event_handler_chapter#event_handling">event handlers</seealso> including start arguments.</item> + <item><c><![CDATA[-ct_hooks <ct_hooks>]]></c>, to install + <seealso marker="ct_hooks_chapter#installing">Common Test Hooks</seealso> including start arguments.</item> <item><c><![CDATA[-include]]></c>, specifies include directories (see above).</item> <item><c><![CDATA[-no_auto_compile]]></c>, disables the automatic test suite compilation feature (see above).</item> <item><c><![CDATA[-multiply_timetraps <n>]]></c>, extends <seealso marker="write_test_chapter#timetraps">timetrap @@ -167,20 +170,20 @@ <note><p>Directories passed to Common Test may have either relative or absolute paths.</p></note> <note><p>Arbitrary start flags to the Erlang Runtime System may also be passed as - parameters to <c>run_test</c>. It is, for example, useful to be able to + parameters to <c>ct_run</c>. It is, for example, useful to be able to pass directories that should be added to the Erlang code server search path with the <c>-pa</c> or <c>-pz</c> flag. If you have common help- or library modules for test suites (separately compiled), stored in other directories than the test suite directories, these help/lib directories are preferrably added to the code path this way. Example:</p> - <p><c>$ run_test -dir ./chat_server -logdir ./chat_server/testlogs -pa $PWD/chat_server/ebin</c></p> + <p><c>$ ct_run -dir ./chat_server -logdir ./chat_server/testlogs -pa $PWD/chat_server/ebin</c></p> <p>Note how in this example, the absolute path of the <c>chat_server/ebin</c> directory is passed to the code server. This is essential since relative paths are stored by the code server as relative, and Common Test changes the current working directory of the Erlang Runtime System during the test run!</p> </note> - <p>For more information about the <c>run_test</c> program, see the + <p>For more information about the <c>ct_run</c> program, see the <seealso marker="install_chapter#general">Installation</seealso> chapter. </p> </section> @@ -188,7 +191,7 @@ <section> <title>Running tests from the Web based GUI</title> - <p>The web based GUI, VTS, is started with the <c>run_test</c> + <p>The web based GUI, VTS, is started with the <c>ct_run</c> program. From the GUI you can load config files, and select directories, suites and cases to run. You can also state the config files, directories, suites and cases on the command line @@ -196,22 +199,22 @@ </p> <list> - <item><c>run_test -vts</c></item> - <item><c><![CDATA[run_test -vts -config <configfilename>]]></c></item> - <item><c><![CDATA[run_test -vts -config <configfilename> -suite <suitewithfullpath> + <item><c>ct_run -vts</c></item> + <item><c><![CDATA[ct_run -vts -config <configfilename>]]></c></item> + <item><c><![CDATA[ct_run -vts -config <configfilename> -suite <suitewithfullpath> -case <casename>]]></c></item> </list> <p>From the GUI you can run tests and view the result and the logs. </p> - <p>Note that <c>run_test -vts</c> will try to open the Common Test start + <p>Note that <c>ct_run -vts</c> will try to open the Common Test start page in an existing web browser window or start the browser if it is not running. Which browser should be started may be specified with the browser start command option:</p> - <p><c><![CDATA[run_test -vts -browser <browser_start_cmd>]]></c></p> + <p><c><![CDATA[ct_run -vts -browser <browser_start_cmd>]]></c></p> <p>Example:</p> - <p><c><![CDATA[$ run_test -vts -browser 'firefox&']]></c></p> + <p><c><![CDATA[$ ct_run -vts -browser 'firefox&']]></c></p> <p>Note that the browser must run as a separate OS process or VTS will hang!</p> <p>If no specific browser start command is specified, Firefox will be the default browser on Unix platforms and Internet Explorer on Windows. @@ -227,10 +230,10 @@ <p>Common Test provides an Erlang API for running tests. The main (and most flexible) function for specifying and executing tests is called <c>ct:run_test/1</c>. This function takes the same start parameters as - the <c>run_test</c> program described above, only the flags are instead + the <c>ct_run</c> program described above, only the flags are instead given as options in a list of key-value tuples. E.g. a test specified - with <c>run_test</c> like:</p> - <p><c>$ run_test -suite ./my_SUITE -logdir ./results</c></p> + with <c>ct_run</c> like:</p> + <p><c>$ ct_run -suite ./my_SUITE -logdir ./results</c></p> <p>is with <c>ct:run_test/1</c> specified as:</p> <p><c>1> ct:run_test([{suite,"./my_SUITE"},{logdir,"./results"}]).</c></p> <p>For detailed documentation, please see the <c>ct</c> manual page.</p> @@ -253,17 +256,17 @@ manually and call <c>ct:install/1</c> to install any configuration data you might need (use <c>[]</c> as argument otherwise), then call <c>ct:start_interactive/0</c> to start Common Test. If you use - the <c>run_test</c> program, you may start the Erlang shell and Common Test + the <c>ct_run</c> program, you may start the Erlang shell and Common Test in the same go by using the <c>-shell</c> and, optionally, the <c>-config</c> and/or <c>-userconfig</c> flag. Examples: </p> <list> - <item><c>run_test -shell</c></item> - <item><c><![CDATA[run_test -shell -config cfg/db.cfg]]></c></item> - <item><c><![CDATA[run_test -shell -userconfig db_login testuser x523qZ]]></c></item> + <item><c>ct_run -shell</c></item> + <item><c><![CDATA[ct_run -shell -config cfg/db.cfg]]></c></item> + <item><c><![CDATA[ct_run -shell -userconfig db_login testuser x523qZ]]></c></item> </list> - <p>If no config file is given with the <c>run_test</c> command, + <p>If no config file is given with the <c>ct_run</c> command, a warning will be displayed. If Common Test has been run from the same directory earlier, the same config file(s) will be used again. If Common Test has not been run from this directory before, no @@ -293,7 +296,7 @@ <c>ctlog.html</c> in the <c><![CDATA[ct_run.<timestamp>]]></c> directory. A link to this file will be available in the file named <c>last_interactive.html</c> in the directory from which - you executed <c>run_test</c>. Currently, specifying a different + you executed <c>ct_run</c>. Currently, specifying a different root directory for the logs than the current working directory, is not supported.</p> @@ -309,7 +312,7 @@ <section> <title>Step by step execution of test cases with the Erlang Debugger</title> - <p>By means of <c>run_test -step [opts]</c>, or by passing the + <p>By means of <c>ct_run -step [opts]</c>, or by passing the <c>{step,Opts}</c> option to <c>ct:run_test/1</c>, it is possible to get the Erlang Debugger started automatically and use its graphical interface to investigate the state of the current test @@ -333,8 +336,8 @@ with <c>dir</c>.</p> </section> + <marker id="test_specifications"></marker> <section> - <marker id="test_specifications"></marker> <title>Using test specifications</title> <p>The most flexible way to specify what to test, is to use a so @@ -345,12 +348,12 @@ for <c>ct</c>). There are two general types of terms: configuration terms and test specification terms.</p> <p>With configuration terms it is possible to e.g. label the test - run (similar to <c>run_test -label</c>), evaluate arbitrary expressions + run (similar to <c>ct_run -label</c>), evaluate arbitrary expressions before starting a test, import configuration data (similar to - <c>run_test -config/-userconfig</c>), specify HTML log directories (similar + <c>ct_run -config/-userconfig</c>), specify HTML log directories (similar to - <c>run_test -logdir</c>), give aliases to test nodes and test + <c>ct_run -logdir</c>), give aliases to test nodes and test directories (to make a specification easier to read and maintain), enable code coverage analysis (see the <seealso marker="cover_chapter#cover">Code Coverage @@ -359,21 +362,23 @@ Event Handling</seealso> chapter). There is also a term for specifying include directories that should be passed on to the compiler when automatic compilation is performed (similar - to <c>run_test -include</c>, see above).</p> + to <c>ct_run -include</c>, see above).</p> <p>With test specification terms it is possible to state exactly which tests should run and in which order. A test term specifies either one or more suites, one or more test case groups, or one or more test cases in a group or suite.</p> <p>An arbitrary number of test terms may be declared in sequence. - Common Test will compile the terms into one or more tests to be - performed in one resulting test run. Note that a term that + Common Test will by default compile the terms into one or more tests + to be performed in one resulting test run. Note that a term that specifies a set of test cases will "swallow" one that only specifies a subset of these cases. E.g. the result of merging one term that specifies that all cases in suite S should be executed, with another term specifying only test case X and Y in S, is a test of all cases in S. However, if a term specifying test case X and Y in S is merged with a term specifying case Z - in S, the result is a test of X, Y and Z in S.</p> + in S, the result is a test of X, Y and Z in S. To disable this + behaviour, it is possible in test specification to set the + <c>merge_tests</c> term to <c>false</c>.</p> <p>A test term can also specify one or more test suites, groups, or test cases to be skipped. Skipped suites, groups and cases are not executed and show up in the HTML test log files as @@ -432,6 +437,8 @@ {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}. {alias, DirAlias, Dir}. + + {merge_tests, Bool}. {logdir, LogDir}. {logdir, NodeRefs, LogDir}. @@ -440,6 +447,9 @@ {event_handler, NodeRefs, EventHandlers}. {event_handler, EventHandlers, InitArgs}. {event_handler, NodeRefs, EventHandlers, InitArgs}. + + {ct_hooks, CTHModules}. + {ct_hooks, NodeRefs, CTHModules}. </pre> <p>Test terms:</p> <pre> @@ -478,6 +488,9 @@ LogDir = string() EventHandlers = atom() | [atom()] InitArgs = [term()] + CTHModules = [CTHModule | {CTHModule, CTHInitArgs}] + CTHModule = atom() + CTHInitArgs = term() DirRef = DirAlias | Dir Suites = atom() | [atom()] | all Suite = atom() @@ -535,7 +548,7 @@ <p>It is possible for the user to provide a test specification that includes (for Common Test) unrecognizable terms. If this is desired, the <c>-allow_user_terms</c> flag should be used when starting tests with - <c>run_test</c>. This forces Common Test to ignore unrecognizable terms. + <c>ct_run</c>. This forces Common Test to ignore unrecognizable terms. Note that in this mode, Common Test is not able to check the specification for errors as efficiently as if the scanner runs in default mode. If <c>ct:run_test/1</c> is used for starting the tests, the relaxed scanner @@ -661,11 +674,11 @@ </pre> <p>To install the CSS file (Common Test inlines the definition in the - HTML code), the name may be provided when executing <c>run_test</c>. + HTML code), the name may be provided when executing <c>ct_run</c>. Example:</p> <pre> - $ run_test -dir $TEST/prog -stylesheet $TEST/styles/test_categories.css + $ ct_run -dir $TEST/prog -stylesheet $TEST/styles/test_categories.css </pre> <p>Categories in a CSS file installed with the <c>-stylesheet</c> flag @@ -738,7 +751,7 @@ means of time, it is also possible to specify what action Common Test should take upon timeout. Either Common Test performs all tests in the current run before stopping, or it stops as soon as the current test job is finished. Repetition can be activated by - means of <c>run_test</c> start flags, or tuples in the <c>ct:run:test/1</c> + means of <c>ct_run</c> start flags, or tuples in the <c>ct:run:test/1</c> option list argument. The flags (options in parenthesis) are:</p> <list> <item><c>-repeat N ({repeat,N})</c>, where <c>N</c> is a positive integer.</item> @@ -774,7 +787,7 @@ <p>Example 1:</p> <pre> - $ run_test -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -duration 001000 -force_stop</pre> + $ ct_run -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -duration 001000 -force_stop</pre> <p>Here the suites in test directory to1, followed by the suites in to2, will be executed in one test run. A timeout event will occur after 10 minutes. As long as there is time left, Common Test will repeat the test run (i.e. starting over with the to1 test). @@ -787,7 +800,7 @@ $ date Fri Sep 28 15:00:00 MEST 2007 - $ run_test -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -until 160000</pre> + $ ct_run -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -until 160000</pre> <p>Here the same test run as in the example above will be executed (and possibly repeated). In this example, however, the timeout will occur after 1 hour and when that happens, Common Test will finish the entire test run before stopping (i.e. the to1 and to2 test @@ -795,7 +808,7 @@ <p>Example 3:</p> <pre> - $ run_test -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -repeat 5</pre> + $ ct_run -dir $TEST_ROOT/to1 $TEST_ROOT/to2 -repeat 5</pre> <p>Here the test run, including both the to1 and the to2 test, will be repeated 5 times.</p> <note><p>This feature should not be confused with the <c>repeat</c> property of a test @@ -814,7 +827,7 @@ of the <c>-silent_connections</c> flag:</p> <pre> - run_test -silent_connections [conn_types] + ct_run -silent_connections [conn_types] </pre> <p>where <c>conn_types</c> specifies <c>telnet, ftp, rpc</c> and/or <c>snmp</c>.</p> @@ -822,11 +835,11 @@ <p>Example:</p> <pre> - run_test ... -silent_connections telnet ftp</pre> + ct_run ... -silent_connections telnet ftp</pre> <p>switches off logging for telnet and ftp connections.</p> <pre> - run_test ... -silent_connections</pre> + ct_run ... -silent_connections</pre> <p>switches off logging for all connection types.</p> diff --git a/lib/common_test/doc/src/test_structure_chapter.xml b/lib/common_test/doc/src/test_structure_chapter.xml index cd38ae0c7c..b9ca59135d 100644 --- a/lib/common_test/doc/src/test_structure_chapter.xml +++ b/lib/common_test/doc/src/test_structure_chapter.xml @@ -144,7 +144,7 @@ be used when the test suite needs to write to files. </item> - <tag><em>run_test</em></tag> + <tag><em>ct_run</em></tag> <item> The name of an executable program that may be used as an interface for specifying and running diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 5afec6de6a..723492d8f3 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2010</year> + <year>2003</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -115,6 +115,7 @@ </p> </section> + <marker id="per_testcase"/> <section> <title>Init and end per test case</title> diff --git a/lib/common_test/priv/Makefile.in b/lib/common_test/priv/Makefile.in index a6ac0f1a02..f4a0c181f9 100644 --- a/lib/common_test/priv/Makefile.in +++ b/lib/common_test/priv/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2009. All Rights Reserved. +# Copyright Ericsson AB 2003-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 027667e6b0..84b122b5e4 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# Copyright Ericsson AB 2003-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -67,7 +67,9 @@ MODULES= \ ct_config \ ct_config_plain \ ct_config_xml \ - ct_slave + ct_slave \ + ct_hooks\ + ct_hooks_lock TARGET_MODULES= $(MODULES:%=$(EBIN)/%) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 1dbf83ee10..dfec2b7a67 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% Copyright Ericsson AB 2003-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -97,7 +97,7 @@ %%% <code>install([{config,["config_node.ctc","config_user.ctc"]}])</code>.</p> %%% %%% <p>Note that this function is automatically run by the -%%% <code>run_test</code> program.</p> +%%% <code>ct_run</code> program.</p> install(Opts) -> ct_run:install(Opts). @@ -148,7 +148,8 @@ run(TestDirs) -> %%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} | %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | -%%% {refresh_logs,LogDir} | {basic_html,Bool} +%%% {refresh_logs,LogDir} | {basic_html,Bool} | +%%% {ct_hooks, CTHs} %%% TestDirs = [string()] | string() %%% Suites = [string()] | string() %%% Cases = [atom()] | atom() @@ -176,13 +177,16 @@ run(TestDirs) -> %%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile} %%% DecryptKey = string() %%% DecryptFile = string() +%%% CTHs = [CTHModule | {CTHModule, CTHInitArgs}] +%%% CTHModule = atom() +%%% CTHInitArgs = term() %%% Result = [TestResult] | {error,Reason} %%% @doc Run tests as specified by the combination of options in <code>Opts</code>. %%% The options are the same as those used with the -%%% <seealso marker="run_test#run_test"><code>run_test</code></seealso> program. +%%% <seealso marker="ct_run#ct_run"><code>ct_run</code></seealso> program. %%% Note that here a <code>TestDir</code> can be used to point out the path to %%% a <code>Suite</code>. Note also that the option <code>testcase</code> -%%% corresponds to the <code>-case</code> option in the <code>run_test</code> +%%% corresponds to the <code>-case</code> option in the <code>ct_run</code> %%% program. Configuration files specified in <code>Opts</code> will be %%% installed automatically at startup. run_test(Opts) -> @@ -225,7 +229,7 @@ step(TestDir,Suite,Case,Opts) -> %%% %%% <p>From this mode all test case support functions can be executed %%% directly from the erlang shell. The interactive mode can also be -%%% started from the OS command line with <code>run_test -shell +%%% started from the OS command line with <code>ct_run -shell %%% [-config File...]</code>.</p> %%% %%% <p>If any functions using "required config data" (e.g. telnet or diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index f2ca023cff..38a2aa53ac 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,7 @@ -module(ct_framework). --export([init_tc/3, end_tc/3, get_suite/2, report/2, warn/1]). +-export([init_tc/3, end_tc/4, get_suite/2, report/2, warn/1]). -export([error_notification/4]). -export([overview_html_header/1]). @@ -207,7 +207,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; - FinalConfig -> + {ok, FinalConfig} -> case MergeResult of {error,Reason} -> %% suite0 configure finished now, report that @@ -216,13 +216,25 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> _ -> case get('$test_server_framework_test') of undefined -> - FinalConfig; + ct_suite_init(Mod, FuncSpec, FinalConfig); Fun -> - Fun(init_tc, FinalConfig) + case Fun(init_tc, FinalConfig) of + NewConfig when is_list(NewConfig) -> + {ok,NewConfig}; + Else -> + Else + end end end end. - + +ct_suite_init(Mod, Func, [Config]) when is_list(Config) -> + case ct_hooks:init_tc( Mod, Func, Config) of + NewConfig when is_list(NewConfig) -> + {ok, [NewConfig]}; + Else -> + Else + end. add_defaults(Mod,Func,FuncInfo,DoInit) -> case (catch Mod:suite()) of @@ -239,7 +251,9 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> (_) -> false end, SuiteInfo) of true -> - SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfo), + SuiteInfoNoCTH = + lists:keydelete(ct_hooks,1,SuiteInfo), + SuiteInfo1 = merge_with_suite_defaults(Mod,SuiteInfoNoCTH), case add_defaults1(Mod,Func,FuncInfo,SuiteInfo1,DoInit) of Error = {error,_} -> {SuiteInfo1,Error}; MergedInfo -> {SuiteInfo1,MergedInfo} @@ -362,6 +376,8 @@ configure([{timetrap,off}|Rest],Info,SuiteInfo,Scope,Config) -> configure([{timetrap,Time}|Rest],Info,SuiteInfo,Scope,Config) -> Dog = test_server:timetrap(Time), configure(Rest,Info,SuiteInfo,Scope,[{watchdog,Dog}|Config]); +configure([{ct_hooks, Hook} | Rest], Info, SuiteInfo, Scope, Config) -> + configure(Rest, Info, SuiteInfo, Scope, [{ct_hooks, Hook} | Config]); configure([_|Rest],Info,SuiteInfo,Scope,Config) -> configure(Rest,Info,SuiteInfo,Scope,Config); configure([],_,_,_,Config) -> @@ -418,14 +434,14 @@ try_set_default(Name,Key,Info,Where) -> %%% %%% @doc Test server framework callback, called by the test_server %%% when a test case is finished. -end_tc(?MODULE,error_in_suite,_) -> % bad start! +end_tc(?MODULE,error_in_suite,_, _) -> % bad start! ok; -end_tc(Mod,Func,{TCPid,Result,[Args]}) when is_pid(TCPid) -> - end_tc(Mod,Func,TCPid,Result,Args); -end_tc(Mod,Func,{Result,[Args]}) -> - end_tc(Mod,Func,self(),Result,Args). +end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) -> + end_tc(Mod,Func,TCPid,Result,Args,Return); +end_tc(Mod,Func,{Result,[Args]}, Return) -> + end_tc(Mod,Func,self(),Result,Args,Return). -end_tc(Mod,Func,TCPid,Result,Args) -> +end_tc(Mod,Func,TCPid,Result,Args,Return) -> case lists:keysearch(watchdog,1,Args) of {value,{watchdog,Dog}} -> test_server:timetrap_cancel(Dog); false -> ok @@ -448,8 +464,10 @@ end_tc(Mod,Func,TCPid,Result,Args) -> {_,GroupName,_Props} = Group -> case lists:keysearch(save_config,1,Args) of {value,{save_config,SaveConfig}} -> - ct_util:save_suite_data(last_saved_config, - {Mod,{group,GroupName}},SaveConfig), + ct_util:save_suite_data( + last_saved_config, + {Mod,{group,GroupName}}, + SaveConfig), Group; false -> Group @@ -466,12 +484,33 @@ end_tc(Mod,Func,TCPid,Result,Args) -> end, ct_util:reset_silent_connections(), - %% send sync notification so that event handlers may print - %% in the log file before it gets closed - ct_event:sync_notify(#event{name=tc_done, - node=node(), - data={Mod,FuncSpec,tag(Result)}}), - case Result of + case get('$test_server_framework_test') of + undefined -> + {FinalResult,FinalNotify} = + case ct_hooks:end_tc( + Mod, FuncSpec, Args, Result, Return) of + '$ct_no_change' -> + {FinalResult = ok,Result}; + FinalResult -> + {FinalResult,FinalResult} + end, + % send sync notification so that event handlers may print + % in the log file before it gets closed + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={Mod,FuncSpec, + tag_cth(FinalNotify)}}); + Fun -> + % send sync notification so that event handlers may print + % in the log file before it gets closed + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={Mod,FuncSpec,tag(Result)}}), + FinalResult = Fun(end_tc, Return) + end, + + + case FinalResult of {skip,{sequence_failed,_,_}} -> %% ct_logs:init_tc is never called for a skipped test case %% in a failing sequence, so neither should end_tc @@ -490,12 +529,7 @@ end_tc(Mod,Func,TCPid,Result,Args) -> _ -> ok end, - case get('$test_server_framework_test') of - undefined -> - ok; - Fun -> - Fun(end_tc, ok) - end. + FinalResult. %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | @@ -511,6 +545,21 @@ tag(E = testcase_aborted_or_killed) -> tag(Other) -> Other. +tag_cth({STag,Reason}) when STag == skip; STag == skipped -> + {skipped,Reason}; +tag_cth({fail, Reason}) -> + {failed, {error,Reason}}; +tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; + ETag == timetrap_timeout; + ETag == testcase_aborted -> + {failed,E}; +tag_cth(E = testcase_aborted_or_killed) -> + {failed,E}; +tag_cth(List) when is_list(List) -> + ok; +tag_cth(Other) -> + Other. + %%%----------------------------------------------------------------- %%% @spec error_notification(Mod,Func,Args,Error) -> ok %%% Mod = atom() @@ -694,12 +743,12 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> %% init/end functions for top groups will be executed case catch proplists:get_value(name, element(2, hd(ConfTests))) of Name -> % top group - ConfTests; + delete_subs(ConfTests, ConfTests); _ -> [] end; false -> - ConfTests + delete_subs(ConfTests, ConfTests) end end; _ -> @@ -716,9 +765,25 @@ get_suite(Mod, Name) -> find_groups(Mod, Name, TCs, GroupDefs) -> Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), Trimmed = trim(Found), - delete_subs(Trimmed, Trimmed). - -find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) -> + %% I cannot find a reason to why this function is called, + %% It deletes any group which is referenced in any other + %% group. i.e. + %% groups() -> + %% [{test, [], [testcase1]}, + %% {testcases, [], [{group, test}]}]. + %% Would be changed to + %% groups() -> + %% [{testcases, [], [testcase1]}]. + %% instead of what I believe is correct: + %% groups() -> + %% [{test, [], [testcase1]}, + %% {testcases, [], [testcase1]}]. + %% Have to double check with peppe + delete_subs(Trimmed, Trimmed), + Trimmed. + +find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) + when is_atom(Name), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name, Known), [make_conf(Mod, Name, Props, find(Mod, all, all, Tests, [Name | Known], Defs, true)) | @@ -740,8 +805,8 @@ find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false) find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false) when is_atom(Name1), is_list(Props), is_list(Tests) -> cyclic_test(Mod, Name1, Known), - [make_conf(Mod, Name1, Props, - find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | + [make_conf(Mod,Name1,Props, + find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | find(Mod, Name, TCs, Gs, [], Defs, false)]; find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true) @@ -757,17 +822,31 @@ find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true) find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) | find(Mod, Name, all, Gs, [], Defs, true)]; -find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) when is_atom(Name1) -> +find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) + when is_atom(Name1) -> find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found); +%% Undocumented remote group feature, use with caution +find(Mod, Name, TCs, [{group, ExtMod, ExtGrp} | Gs], Known, Defs, true) + when is_atom(ExtMod), is_atom(ExtGrp) -> + ExternalDefs = ExtMod:groups(), + ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}], + [], ExternalDefs, false), + ExternalTCs ++ find(Mod, Name, TCs, Gs, Known, Defs, true); + find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found) when is_atom(Name1), is_list(Tests) -> find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found); -find(Mod, Name, TCs, [TC | Gs], Known, Defs, false) when is_atom(TC) -> +find(Mod, Name, TCs, [_TC | Gs], Known, Defs, false) -> find(Mod, Name, TCs, Gs, Known, Defs, false); find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) -> + [{Mod, TC} | find(Mod, Name, TCs, Gs, Known, Defs, true)]; + +find(Mod, Name, TCs, [{ExternalTC, Case} = TC | Gs], Known, Defs, true) + when is_atom(ExternalTC), + is_atom(Case) -> [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)]; find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) -> @@ -787,7 +866,7 @@ find(_Mod, _Name, _TCs, [], _Known, _Defs, false) -> find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) -> []. -delete_subs([Conf | Confs], All) -> +delete_subs([{conf, _,_,_,_} = Conf | Confs], All) -> All1 = delete_conf(Conf, All), case is_sub(Conf, All1) of true -> @@ -795,7 +874,8 @@ delete_subs([Conf | Confs], All) -> false -> delete_subs(Confs, All) end; - +delete_subs([_Else | Confs], All) -> + delete_subs(Confs, All); delete_subs([], All) -> All. @@ -887,7 +967,9 @@ make_all_conf(Mod) -> [] -> {error,{invalid_group_spec,Mod}}; ConfTests -> - [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests] + [{conf,Props,Init,all,End} || + {conf,Props,Init,_,End} + <- delete_subs(ConfTests, ConfTests)] end end. @@ -933,31 +1015,11 @@ get_all(Mod, ConfTests) -> [{?MODULE,error_in_suite,[[{error,What}]]}]; SeqsAndTCs -> %% expand group references in all() using ConfTests - Expand = - fun({group,Name}) -> - FindConf = - fun({conf,Props,_,_,_}) -> - case proplists:get_value(name, Props) of - Name -> true; - _ -> false - end - end, - case lists:filter(FindConf, ConfTests) of - [ConfTest|_] -> - ConfTest; - [] -> - E = "Invalid reference to group "++ - atom_to_list(Name)++" in "++ - atom_to_list(Mod)++":all/0", - throw({error,list_to_atom(E)}) - end; - (SeqOrTC) -> SeqOrTC - end, - case catch lists:map(Expand, SeqsAndTCs) of + case catch expand_groups(SeqsAndTCs, ConfTests, Mod) of {error,_} = Error -> [{?MODULE,error_in_suite,[[Error]]}]; Tests -> - Tests + delete_subs(Tests, Tests) end end; Skip = {skip,_Reason} -> @@ -968,6 +1030,30 @@ get_all(Mod, ConfTests) -> [{?MODULE,error_in_suite,[[{error,Reason}]]}] end. +expand_groups([H | T], ConfTests, Mod) -> + [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]; +expand_groups([], _ConfTests, _Mod) -> + []; +expand_groups({group,Name}, ConfTests, Mod) -> + FindConf = + fun({conf,Props,_,_,_}) -> + case proplists:get_value(name, Props) of + Name -> true; + _ -> false + end + end, + case lists:filter(FindConf, ConfTests) of + [ConfTest|_] -> + expand_groups(ConfTest, ConfTests, Mod); + [] -> + E = "Invalid reference to group "++ + atom_to_list(Name)++" in "++ + atom_to_list(Mod)++":all/0", + throw({error,list_to_atom(E)}) + end; +expand_groups(SeqOrTC, _ConfTests, _Mod) -> + SeqOrTC. + %%!============================================================ %%! The support for sequences by means of using sequences/0 @@ -1137,6 +1223,18 @@ report(What,Data) -> ok; tc_done -> {_Suite,Case,Result} = Data, + case Result of + {failed, _} -> + ct_hooks:on_tc_fail(What, Data); + {skipped,{failed,{_,init_per_testcase,_}}} -> + ct_hooks:on_tc_skip(tc_auto_skip, Data); + {skipped,{require_failed,_}} -> + ct_hooks:on_tc_skip(tc_auto_skip, Data); + {skipped,_} -> + ct_hooks:on_tc_skip(tc_user_skip, Data); + _Else -> + ok + end, case {Case,Result} of {init_per_suite,_} -> ok; @@ -1154,8 +1252,8 @@ report(What,Data) -> add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); - {_,{FailOrSkip,_Reason}} -> - add_to_stats(FailOrSkip) + {_,{SkipOrFail,_Reason}} -> + add_to_stats(SkipOrFail) end; tc_user_skip -> %% test case specified as skipped in testspec @@ -1163,6 +1261,7 @@ report(What,Data) -> ct_event:sync_notify(#event{name=tc_user_skip, node=node(), data=Data}), + ct_hooks:on_tc_skip(What, Data), add_to_stats(user_skipped); tc_auto_skip -> %% test case skipped because of error in init_per_suite @@ -1175,6 +1274,7 @@ report(What,Data) -> ct_event:sync_notify(#event{name=tc_auto_skip, node=node(), data=Data}), + ct_hooks:on_tc_skip(What, Data), if Case /= end_per_suite, Case /= end_per_group -> add_to_stats(auto_skipped); true -> diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl new file mode 100644 index 0000000000..5eddefffce --- /dev/null +++ b/lib/common_test/src/ct_hooks.erl @@ -0,0 +1,307 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework test execution control module. +%%% +%%% <p>This module is a proxy for calling and handling common test hooks.</p> + +-module(ct_hooks). + +%% API Exports +-export([init/1]). +-export([init_tc/3]). +-export([end_tc/5]). +-export([terminate/1]). +-export([on_tc_skip/2]). +-export([on_tc_fail/2]). + +-type proplist() :: [{atom(),term()}]. + +%% If you change this, remember to update ct_util:look -> stop clause as well. +-define(config_name, ct_hooks). + +%% ------------------------------------------------------------------------- +%% API Functions +%% ------------------------------------------------------------------------- + +%% @doc Called before any suites are started +-spec init(State :: term()) -> ok | + {error, Reason :: term()}. +init(Opts) -> + call([{Hook, call_id, undefined} || Hook <- get_new_hooks(Opts)], + ok, init, []). + + +%% @doc Called after all suites are done. +-spec terminate(Hooks :: term()) -> + ok. +terminate(Hooks) -> + call([{HookId, fun call_terminate/3} || {HookId,_,_} <- Hooks], + ct_hooks_terminate_dummy, terminate, Hooks), + ok. + +%% @doc Called as each test case is started. This includes all configuration +%% tests. +-spec init_tc(Mod :: atom(), Func :: atom(), Args :: list()) -> + NewConfig :: proplist() | + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()}. +init_tc(ct_framework, _Func, Args) -> + Args; +init_tc(Mod, init_per_suite, Config) -> + Info = try proplists:get_value(ct_hooks, Mod:suite(),[]) of + List when is_list(List) -> + [{ct_hooks,List}]; + CTHook when is_atom(CTHook) -> + [{ct_hooks,[CTHook]}] + catch error:undef -> + [{ct_hooks,[]}] + end, + call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]); +init_tc(Mod, end_per_suite, Config) -> + call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); +init_tc(Mod, {init_per_group, GroupName, Opts}, Config) -> + maybe_start_locker(Mod, GroupName, Opts), + call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); +init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> + call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); +init_tc(_Mod, TC, Config) -> + call(fun call_generic/3, Config, [pre_init_per_testcase, TC]). + +%% @doc Called as each test case is completed. This includes all configuration +%% tests. +-spec end_tc(Mod :: atom(), + Func :: atom(), + Args :: list(), + Result :: term(), + Resturn :: term()) -> + NewConfig :: proplist() | + {skip, Reason :: term()} | + {auto_skip, Reason :: term()} | + {fail, Reason :: term()} | + ok | '$ct_no_change'. +end_tc(ct_framework, _Func, _Args, Result, _Return) -> + Result; + +end_tc(Mod, init_per_suite, Config, _Result, Return) -> + call(fun call_generic/3, Return, [post_init_per_suite, Mod, Config], + '$ct_no_change'); + +end_tc(Mod, end_per_suite, Config, Result, _Return) -> + call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config], + '$ct_no_change'); + +end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> + call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], + '$ct_no_change'); + +end_tc(Mod, {end_per_group, GroupName, Opts}, Config, Result, _Return) -> + Res = call(fun call_generic/3, Result, + [post_end_per_group, GroupName, Config], '$ct_no_change'), + maybe_stop_locker(Mod, GroupName,Opts), + Res; + +end_tc(_Mod, TC, Config, Result, _Return) -> + call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], + '$ct_no_change'). + +on_tc_skip(How, {_Suite, Case, Reason}) -> + call(fun call_cleanup/3, {How, Reason}, [on_tc_skip, Case]). + +on_tc_fail(_How, {_Suite, Case, Reason}) -> + call(fun call_cleanup/3, Reason, [on_tc_fail, Case]). + +%% ------------------------------------------------------------------------- +%% Internal Functions +%% ------------------------------------------------------------------------- +call_id(Mod, Config, Meta) when is_atom(Mod) -> + call_id({Mod, []}, Config, Meta); +call_id({Mod, Opts}, Config, Scope) -> + Id = catch_apply(Mod,id,[Opts], make_ref()), + {Config, {Id, scope(Scope), {Mod, {Id,Opts}}}}. + +call_init({Mod,{Id,Opts}},Config,_Meta) -> + NewState = Mod:init(Id, Opts), + {Config, {Mod, NewState}}. + +call_terminate({Mod, State}, _, _) -> + catch_apply(Mod,terminate,[State], ok), + {[],{Mod,State}}. + +call_cleanup({Mod, State}, Reason, [Function | Args]) -> + NewState = catch_apply(Mod,Function, Args ++ [Reason, State], + State), + {Reason, {Mod, NewState}}. + +call_generic({Mod, State}, Value, [Function | Args]) -> + {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], + {Value,State}), + {NewValue, {Mod, NewState}}. + +%% Generic call function +call(Fun, Config, Meta) -> + maybe_lock(), + Hooks = get_hooks(), + Res = call([{HookId,Fun} || {HookId,_, _} <- Hooks] ++ + get_new_hooks(Config, Fun), + remove(?config_name,Config), Meta, Hooks), + maybe_unlock(), + Res. + +call(Fun, Config, Meta, NoChangeRet) when is_function(Fun) -> + case call(Fun,Config,Meta) of + Config -> NoChangeRet; + NewReturn -> NewReturn + end; + +call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> + try + {Config, {NewId, _, _} = NewHook} = call_id(Hook, Config, Meta), + {NewHooks, NewRest} = + case lists:keyfind(NewId, 1, Hooks) of + false when NextFun =:= undefined -> + {Hooks ++ [NewHook], + [{NewId, fun call_init/3} | Rest]}; + ExistingHook when is_tuple(ExistingHook) -> + {Hooks, Rest}; + _ -> + {Hooks ++ [NewHook], + [{NewId, fun call_init/3},{NewId,NextFun} | Rest]} + end, + call(NewRest, Config, Meta, NewHooks) + catch Error:Reason -> + Trace = erlang:get_stacktrace(), + ct_logs:log("Suite Hook","Failed to start a CTH: ~p:~p", + [Error,{Reason,Trace}]), + call([], {fail,"Failed to start CTH" + ", see the CT Log for details"}, Meta, Hooks) + end; +call([{HookId, Fun} | Rest], Config, Meta, Hooks) -> + try + {_,Scope,ModState} = lists:keyfind(HookId, 1, Hooks), + {NewConf, NewHookInfo} = Fun(ModState, Config, Meta), + NewCalls = get_new_hooks(NewConf, Fun), + NewHooks = lists:keyreplace(HookId, 1, Hooks, {HookId, Scope, NewHookInfo}), + call(NewCalls ++ Rest, remove(?config_name, NewConf), Meta, + terminate_if_scope_ends(HookId, Meta, NewHooks)) + catch throw:{error_in_cth_call,Reason} -> + call(Rest, {fail, Reason}, Meta, + terminate_if_scope_ends(HookId, Meta, Hooks)) + end; +call([], Config, _Meta, Hooks) -> + save_suite_data_async(Hooks), + Config. + +remove(Key,List) when is_list(List) -> + [Conf || Conf <- List, is_tuple(Conf) =:= false + orelse element(1, Conf) =/= Key]; +remove(_, Else) -> + Else. + +%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc +scope([pre_init_per_testcase, TC|_]) -> + [post_end_per_testcase, TC]; +scope([pre_init_per_group, GroupName|_]) -> + [post_end_per_group, GroupName]; +scope([post_init_per_group, GroupName|_]) -> + [post_end_per_group, GroupName]; +scope([pre_init_per_suite, SuiteName|_]) -> + [post_end_per_suite, SuiteName]; +scope([post_init_per_suite, SuiteName|_]) -> + [post_end_per_suite, SuiteName]; +scope(init) -> + none. + +terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] -> + terminate_if_scope_ends(HookId,[Function,Tag],Hooks); +terminate_if_scope_ends(HookId, Function, Hooks) -> + case lists:keyfind(HookId, 1, Hooks) of + {HookId, Function, _ModState} = Hook -> + terminate([Hook]), + lists:keydelete(HookId, 1, Hooks); + _ -> + Hooks + end. + +%% Fetch hook functions +get_new_hooks(Config, Fun) -> + lists:foldl(fun(NewHook, Acc) -> + [{NewHook, call_id, Fun} | Acc] + end, [], get_new_hooks(Config)). + +get_new_hooks(Config) when is_list(Config) -> + lists:flatmap(fun({?config_name, HookConfigs}) -> + HookConfigs; + (_) -> + [] + end, Config); +get_new_hooks(_Config) -> + []. + +save_suite_data_async(Hooks) -> + ct_util:save_suite_data_async(?config_name, Hooks). + +get_hooks() -> + ct_util:read_suite_data(?config_name). + +catch_apply(M,F,A, Default) -> + try + apply(M,F,A) + catch error:Reason -> + case erlang:get_stacktrace() of + %% Return the default if it was the CTH module which did not have the function. + [{M,F,A}|_] when Reason == undef -> + Default; + Trace -> + ct_logs:log("Suite Hook","Call to CTH failed: ~p:~p", + [error,{Reason,Trace}]), + throw({error_in_cth_call, + lists:flatten( + io_lib:format("~p:~p/~p CTH call failed", + [M,F,length(A)]))}) + end + end. + + +%% We need to lock around the state for parallel groups only. This is because +%% we will get several processes reading and writing the state for a single +%% cth at the same time. +maybe_start_locker(Mod,GroupName,Opts) -> + case lists:member(parallel,Opts) of + true -> + {ok, _Pid} = ct_hooks_lock:start({Mod,GroupName}); + false -> + ok + end. + +maybe_stop_locker(Mod,GroupName,Opts) -> + case lists:member(parallel,Opts) of + true -> + stopped = ct_hooks_lock:stop({Mod,GroupName}); + false -> + ok + end. + + +maybe_lock() -> + locked = ct_hooks_lock:request(). + +maybe_unlock() -> + unlocked = ct_hooks_lock:release(). diff --git a/lib/common_test/src/ct_hooks_lock.erl b/lib/common_test/src/ct_hooks_lock.erl new file mode 100644 index 0000000000..e33fa278dc --- /dev/null +++ b/lib/common_test/src/ct_hooks_lock.erl @@ -0,0 +1,132 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework test execution control module. +%%% +%%% <p>This module is a proxy for calling and handling locks in +%%% common test hooks.</p> + +-module(ct_hooks_lock). + +-behaviour(gen_server). + +%% API +-export([start/1, stop/1, request/0, release/0]). + +%% gen_server callbacks +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +-define(SERVER, ?MODULE). + +-record(state, { id, locked = false, requests = [] }). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc Starts the server +start(Id) -> + case gen_server:start({local, ?SERVER}, ?MODULE, Id, []) of + {error,{already_started, Pid}} -> + {ok,Pid}; + Else -> + Else + end. + +stop(Id) -> + try + gen_server:call(?SERVER, {stop,Id}) + catch exit:{noproc,_} -> + stopped + end. + +request() -> + try + gen_server:call(?SERVER,{request,self()},infinity) + catch exit:{noproc,_} -> + locked + end. + +release() -> + try + gen_server:call(?SERVER,{release,self()}) + catch exit:{noproc,_} -> + unlocked + end. + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +%% @doc Initiates the server +init(Id) -> + {ok, #state{ id = Id }}. + +%% @doc Handling call messages +handle_call({stop,Id}, _From, #state{ id = Id, requests = Reqs } = State) -> + [gen_server:reply(Req, locker_stopped) || {Req,_ReqId} <- Reqs], + {stop, normal, stopped, State}; +handle_call({stop,_Id}, _From, State) -> + {reply, stopped, State}; +handle_call({request, Pid}, _From, #state{ locked = false, + requests = [] } = State) -> + Ref = monitor(process, Pid), + {reply, locked, State#state{ locked = {true, Pid, Ref}} }; +handle_call({request, Pid}, From, #state{ requests = Reqs } = State) -> + {noreply, State#state{ requests = Reqs ++ [{From,Pid}] }}; +handle_call({release, Pid}, _From, #state{ locked = {true, Pid, Ref}, + requests = []} = State) -> + demonitor(Ref,[flush]), + {reply, unlocked, State#state{ locked = false }}; +handle_call({release, Pid}, _From, + #state{ locked = {true, Pid, Ref}, + requests = [{NextFrom,NextPid}|Rest]} = State) -> + demonitor(Ref,[flush]), + gen_server:reply(NextFrom,locked), + NextRef = monitor(process, NextPid), + {reply,unlocked,State#state{ locked = {true, NextPid, NextRef}, + requests = Rest } }; +handle_call({release, _Pid}, _From, State) -> + {reply, not_locked, State}. + +%% @doc Handling cast messages +handle_cast(_Msg, State) -> + {noreply, State}. + +%% @doc Handling all non call/cast messages +handle_info({'DOWN',Ref,process,Pid,_}, + #state{ locked = {true, Pid, Ref}, + requests = [{NextFrom,NextPid}|Rest] } = State) -> + gen_server:reply(NextFrom, locked), + NextRef = monitor(process, NextPid), + {noreply,State#state{ locked = {true, NextPid, NextRef}, + requests = Rest } }. + +%% @doc This function is called by a gen_server when it is about to terminate. +terminate(_Reason, _State) -> + ok. + +%% @doc Convert process state when code is changed +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% ------------------------------------------------------------------------- +%% Internal Functions +%% ------------------------------------------------------------------------- diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index 42e4cf08f4..2ea2ba106a 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -101,12 +101,14 @@ run([TS|TestSpecs],AllowUserTerms,InclNodes,ExclNodes) when is_list(TS), TSRec=#testspec{logdir=AllLogDirs, config=StdCfgFiles, userconfig=UserCfgFiles, + include=AllIncludes, init=AllInitOpts, event_handler=AllEvHs} -> AllCfgFiles = {StdCfgFiles, UserCfgFiles}, RunSkipPerNode = ct_testspec:prepare_tests(TSRec), RunSkipPerNode2 = exclude_nodes(ExclNodes,RunSkipPerNode), - run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs,[],[],AllInitOpts,TS1) + run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs, + AllIncludes,[],[],AllInitOpts,TS1) end, [{TS,Result} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)]; run([],_,_,_) -> @@ -163,11 +165,13 @@ run_on_node([TS|TestSpecs],AllowUserTerms,Node) when is_list(TS),is_atom(Node) - TSRec=#testspec{logdir=AllLogDirs, config=StdCfgFiles, init=AllInitOpts, + include=AllIncludes, userconfig=UserCfgFiles, event_handler=AllEvHs} -> AllCfgFiles = {StdCfgFiles, UserCfgFiles}, {Run,Skip} = ct_testspec:prepare_tests(TSRec,Node), - run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs,[],[],AllInitOpts,TS1) + run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs, + AllIncludes, [],[],AllInitOpts,TS1) end, [{TS,Result} | run_on_node(TestSpecs,AllowUserTerms,Node)]; run_on_node([],_,_) -> @@ -189,7 +193,7 @@ run_on_node(TestSpecs,Node) -> run_all([{Node,Run,Skip}|Rest],AllLogDirs, {AllStdCfgFiles, AllUserCfgFiles}=AllCfgFiles, - AllEvHs,NodeOpts,LogDirs,InitOptions,Specs) -> + AllEvHs,AllIncludes,NodeOpts,LogDirs,InitOptions,Specs) -> LogDir = lists:foldl(fun({N,Dir},_Found) when N == Node -> Dir; @@ -211,6 +215,14 @@ run_all([{Node,Run,Skip}|Rest],AllLogDirs, ({_N,_F},Fs) -> Fs; (F,Fs) -> [{userconfig, F}|Fs] end,[],AllUserCfgFiles), + + Includes = lists:foldr(fun({N,I},Acc) when N =:= Node -> + [I|Acc]; + ({_,_},Acc) -> + Acc; + (I,Acc) -> + [I | Acc] + end, [], AllIncludes), EvHs = lists:foldr(fun({N,H,A},Hs) when N == Node -> [{H,A}|Hs]; ({_N,_H,_A},Hs) -> Hs; @@ -219,10 +231,13 @@ run_all([{Node,Run,Skip}|Rest],AllLogDirs, NO = {Node,[{prepared_tests,{Run,Skip},Specs}, {logdir,LogDir}, + {include, Includes}, {config,StdCfgFiles}, {event_handler,EvHs}] ++ UserCfgFiles}, - run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,[NO|NodeOpts],[LogDir|LogDirs],InitOptions,Specs); -run_all([],AllLogDirs,_,AllEvHs,NodeOpts,LogDirs,InitOptions,Specs) -> + run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,AllIncludes, + [NO|NodeOpts],[LogDir|LogDirs],InitOptions,Specs); +run_all([],AllLogDirs,_,AllEvHs,_AllIncludes, + NodeOpts,LogDirs,InitOptions,Specs) -> Handlers = [{H,A} || {Master,H,A} <- AllEvHs, Master == master], MasterLogDir = case lists:keysearch(master,1,AllLogDirs) of {value,{_,Dir}} -> Dir; diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 586b3893f1..7bd7dc7d66 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -54,6 +54,7 @@ logdir, config = [], event_handlers = [], + ct_hooks = [], include = [], silent_connections, stylesheet, @@ -65,12 +66,12 @@ %%%----------------------------------------------------------------- %%% @spec script_start() -> void() %%% -%%% @doc Start tests via the run_test program or script. +%%% @doc Start tests via the ct_run program or script. %%% -%%% <p>Example:<br/><code>./run_test -config config.ctc -dir +%%% <p>Example:<br/><code>./ct_run -config config.ctc -dir %%% $TEST_DIR</code></p> %%% -%%% <p>Example:<br/><code>./run_test -config config.ctc -suite +%%% <p>Example:<br/><code>./ct_run -config config.ctc -suite %%% $SUITE_PATH/$SUITE_NAME [-case $CASE_NAME]</code></p> %%% script_start() -> @@ -80,7 +81,7 @@ script_start() -> (_) -> true end, Init), %% convert relative dirs added with pa or pz (pre erl_args on - %% the run_test command line) to absolute so that app modules + %% the ct_run command line) to absolute so that app modules %% can be found even after CT changes CWD to logdir rel_to_abs(CtArgs), @@ -171,6 +172,7 @@ script_start1(Parent, Args) -> ([]) -> true end, false, Args), EvHandlers = event_handler_args2opts(Args), + CTHooks = ct_hooks_args2opts(Args), %% check flags and set corresponding application env variables @@ -234,6 +236,7 @@ script_start1(Parent, Args) -> StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover, logdir = LogDir, event_handlers = EvHandlers, + ct_hooks = CTHooks, include = IncludeDirs, silent_connections = SilentConns, stylesheet = Stylesheet, @@ -305,6 +308,10 @@ script_start2(StartOpts = #opts{vts = undefined, SpecStartOpts#opts.scale_timetraps), AllEvHs = merge_vals([StartOpts#opts.event_handlers, SpecStartOpts#opts.event_handlers]), + AllCTHooks = merge_vals( + [StartOpts#opts.ct_hooks, + SpecStartOpts#opts.ct_hooks]), + AllInclude = merge_vals([StartOpts#opts.include, SpecStartOpts#opts.include]), application:set_env(common_test, include, AllInclude), @@ -315,6 +322,7 @@ script_start2(StartOpts = #opts{vts = undefined, logdir = LogDir, config = SpecStartOpts#opts.config, event_handlers = AllEvHs, + ct_hooks = AllCTHooks, include = AllInclude, multiply_timetraps = MultTT, scale_timetraps = ScaleTT}} @@ -332,7 +340,8 @@ script_start2(StartOpts = #opts{vts = undefined, {error,no_testspec_specified}; {undefined,_} -> % no testspec used case check_and_install_configfiles(InitConfig, TheLogDir, - Opts#opts.event_handlers) of + Opts#opts.event_handlers, + Opts#opts.ct_hooks) of ok -> % go on read tests from start flags script_start3(Opts#opts{config=InitConfig, logdir=TheLogDir}, Args); @@ -343,7 +352,8 @@ script_start2(StartOpts = #opts{vts = undefined, %% merge config from start flags with config from testspec AllConfig = merge_vals([InitConfig, Opts#opts.config]), case check_and_install_configfiles(AllConfig, TheLogDir, - Opts#opts.event_handlers) of + Opts#opts.event_handlers, + Opts#opts.ct_hooks) of ok -> % read tests from spec {Run,Skip} = ct_testspec:prepare_tests(Terms, node()), do_run(Run, Skip, Opts#opts{config=AllConfig, @@ -358,7 +368,8 @@ script_start2(StartOpts, Args) -> InitConfig = ct_config:prepare_config_list(Args), LogDir = which(logdir, StartOpts#opts.logdir), case check_and_install_configfiles(InitConfig, LogDir, - StartOpts#opts.event_handlers) of + StartOpts#opts.event_handlers, + StartOpts#opts.ct_hooks) of ok -> % go on read tests from start flags script_start3(StartOpts#opts{config=InitConfig, logdir=LogDir}, Args); @@ -366,11 +377,12 @@ script_start2(StartOpts, Args) -> Error end. -check_and_install_configfiles(Configs, LogDir, EvHandlers) -> +check_and_install_configfiles(Configs, LogDir, EvHandlers, CTHooks) -> case ct_config:check_config_files(Configs) of false -> install([{config,Configs}, - {event_handler,EvHandlers}], LogDir); + {event_handler,EvHandlers}, + {ct_hooks,CTHooks}], LogDir); {value,{error,{nofile,File}}} -> {error,{cant_read_config_file,File}}; {value,{error,{wrong_config,Message}}}-> @@ -438,11 +450,13 @@ script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, script_start4(#opts{label = Label, shell = true, config = Config, event_handlers = EvHandlers, + ct_hooks = CTHooks, logdir = LogDir, testspecs = Specs}, _Args) -> %% label - used by ct_logs application:set_env(common_test, test_label, Label), - InstallOpts = [{config,Config},{event_handler,EvHandlers}], + InstallOpts = [{config,Config},{event_handler,EvHandlers}, + {ct_hooks, CTHooks}], if Config == [] -> ok; true -> @@ -482,11 +496,11 @@ script_start4(Opts = #opts{tests = Tests}, Args) -> %%%----------------------------------------------------------------- %%% @spec script_usage() -> ok -%%% @doc Print usage information for <code>run_test</code>. +%%% @doc Print usage information for <code>ct_run</code>. script_usage() -> io:format("\n\nUsage:\n\n"), io:format("Run tests in web based GUI:\n\n" - "\trun_test -vts [-browser Browser]" + "\tct_run -vts [-browser Browser]" "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" "\n\t[-dir TestDir1 TestDir2 .. TestDirN] |" @@ -497,7 +511,7 @@ script_usage() -> "\n\t[-scale_timetraps]" "\n\t[-basic_html]\n\n"), io:format("Run tests from command line:\n\n" - "\trun_test [-dir TestDir1 TestDir2 .. TestDirN] |" + "\tct_run [-dir TestDir1 TestDir2 .. TestDirN] |" "\n\t[-suite Suite1 Suite2 .. SuiteN [-case Case1 Case2 .. CaseN]]" "\n\t[-step [config | keep_inactive]]" "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" @@ -508,6 +522,7 @@ script_usage() -> "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -517,7 +532,7 @@ script_usage() -> "\n\t[-duration HHMMSS [-force_stop]] |" "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"), io:format("Run tests using test specification:\n\n" - "\trun_test -spec TestSpec1 TestSpec2 .. TestSpecN" + "\tct_run -spec TestSpec1 TestSpec2 .. TestSpecN" "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" "\n\t[-logdir LogDir]" @@ -526,6 +541,7 @@ script_usage() -> "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -535,11 +551,11 @@ script_usage() -> "\n\t[-duration HHMMSS [-force_stop]] |" "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"), io:format("Refresh the HTML index files:\n\n" - "\trun_test -refresh_logs [LogDir]" + "\tct_run -refresh_logs [LogDir]" "[-logdir LogDir] " "[-basic_html]\n\n"), io:format("Run CT in interactive mode:\n\n" - "\trun_test -shell" + "\tct_run -shell" "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]\n\n"). @@ -664,6 +680,9 @@ run_test1(StartOpts) -> end, Hs)) end, + %% CT Hooks + CTHooks = get_start_opt(ct_hooks, value, [], StartOpts), + %% silent connections SilentConns = get_start_opt(silent_connections, fun(all) -> []; @@ -733,7 +752,9 @@ run_test1(StartOpts) -> Opts = #opts{label = Label, cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, - event_handlers = EvHandlers, include = Include, + event_handlers = EvHandlers, + ct_hooks = CTHooks, + include = Include, silent_connections = SilentConns, stylesheet = Stylesheet, multiply_timetraps = MultiplyTT, @@ -784,11 +805,16 @@ run_spec_file(Relaxed, SpecOpts#opts.event_handlers]), AllInclude = merge_vals([Opts#opts.include, SpecOpts#opts.include]), + + AllCTHooks = merge_vals([Opts#opts.ct_hooks, + SpecOpts#opts.ct_hooks]), + application:set_env(common_test, include, AllInclude), case check_and_install_configfiles(AllConfig, which(logdir,LogDir), - AllEvHs) of + AllEvHs, + AllCTHooks) of ok -> Opts1 = Opts#opts{label = Label, cover = Cover, @@ -798,7 +824,8 @@ run_spec_file(Relaxed, include = AllInclude, testspecs = AbsSpecs, multiply_timetraps = MultTT, - scale_timetraps = ScaleTT}, + scale_timetraps = ScaleTT, + ct_hooks = AllCTHooks}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); {error,GCFReason} -> @@ -808,10 +835,12 @@ run_spec_file(Relaxed, run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, config = CfgFiles, - event_handlers = EvHandlers}, + event_handlers = EvHandlers, + ct_hooks = CTHooks}, StartOpts) -> LogDir1 = which(logdir, LogDir), - case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of + case check_and_install_configfiles(CfgFiles, LogDir1, + EvHandlers, CTHooks) of ok -> reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1}, StartOpts)); @@ -842,7 +871,8 @@ check_config_file(Callback, File)-> run_dir(Opts = #opts{logdir = LogDir, config = CfgFiles, - event_handlers = EvHandlers}, StartOpts) -> + event_handlers = EvHandlers, + ct_hooks = CTHook }, StartOpts) -> LogDir1 = which(logdir, LogDir), Opts1 = Opts#opts{logdir = LogDir1}, AbsCfgFiles = @@ -863,7 +893,9 @@ run_dir(Opts = #opts{logdir = LogDir, check_config_file(Callback, File) end, FileList)} end, CfgFiles), - case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir1) of + case install([{config,AbsCfgFiles}, + {event_handler,EvHandlers}, + {ct_hooks, CTHook}], LogDir1) of ok -> ok; {error,IReason} -> exit(IReason) end, @@ -968,7 +1000,8 @@ run_testspec1(TestSpec) -> application:set_env(common_test, include, AllInclude), LogDir1 = which(logdir,Opts#opts.logdir), case check_and_install_configfiles(Opts#opts.config, LogDir1, - Opts#opts.event_handlers) of + Opts#opts.event_handlers, + Opts#opts.ct_hooks) of ok -> Opts1 = Opts#opts{testspecs = [], logdir = LogDir1, @@ -986,6 +1019,7 @@ get_data_for_node(#testspec{label = Labels, config = Cfgs, userconfig = UsrCfgs, event_handler = EvHs, + ct_hooks = CTHooks, include = Incl, multiply_timetraps = MTs, scale_timetraps = STs}, Node) -> @@ -1000,12 +1034,14 @@ get_data_for_node(#testspec{label = Labels, ConfigFiles = [{?ct_config_txt,F} || {N,F} <- Cfgs, N==Node] ++ [CBF || {N,CBF} <- UsrCfgs, N==Node], EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node], + FiltCTHooks = [Hook || {N,Hook} <- CTHooks, N==Node], Include = [I || {N,I} <- Incl, N==Node], #opts{label = Label, logdir = LogDir, cover = Cover, config = ConfigFiles, event_handlers = EvHandlers, + ct_hooks = FiltCTHooks, include = Include, multiply_timetraps = MT, scale_timetraps = ST}. @@ -1036,15 +1072,7 @@ refresh_logs(LogDir) -> which(logdir, undefined) -> "."; which(logdir, Dir) -> - Dir; -which(multiply_timetraps, undefined) -> - 1; -which(multiply_timetraps, MT) -> - MT; -which(scale_timetraps, undefined) -> - false; -which(scale_timetraps, ST) -> - ST. + Dir. choose_val(undefined, V1) -> V1; @@ -2032,12 +2060,37 @@ get_start_opt(Key, IfExists, IfNotExists, Args) -> Val; {value,{Key,_Val}} -> IfExists; - _ when is_function(IfNotExists) -> - IfNotExists(); _ -> IfNotExists end. +ct_hooks_args2opts(Args) -> + ct_hooks_args2opts( + proplists:get_value(ct_hooks, Args, []),[]). + +ct_hooks_args2opts([CTH,Arg,"and"| Rest],Acc) -> + ct_hooks_args2opts(Rest,[{list_to_atom(CTH), + parse_cth_args(Arg)}|Acc]); +ct_hooks_args2opts([CTH], Acc) -> + ct_hooks_args2opts([CTH,"and"],Acc); +ct_hooks_args2opts([CTH, "and" | Rest], Acc) -> + ct_hooks_args2opts(Rest,[list_to_atom(CTH)|Acc]); +ct_hooks_args2opts([CTH, Args], Acc) -> + ct_hooks_args2opts([CTH, Args, "and"],Acc); +ct_hooks_args2opts([],Acc) -> + lists:reverse(Acc). + +parse_cth_args(String) -> + try + true = io_lib:printable_list(String), + {ok,Toks,_} = erl_scan:string(String++"."), + {ok, Args} = erl_parse:parse_term(Toks), + Args + catch _:_ -> + String + end. + + event_handler_args2opts(Args) -> case proplists:get_value(event_handler, Args) of undefined -> @@ -2103,7 +2156,7 @@ get_pa_pz([], PA, PZ) -> {PA,PZ}. %% This function translates ct:run_test/1 start options -%% to run_test start arguments (on the init arguments format) - +%% to ct_run start arguments (on the init arguments format) - %% this is useful mainly for testing the ct_run start functions. opts2args(EnvStartOpts) -> lists:flatmap(fun({config,CfgFiles}) -> @@ -2165,6 +2218,22 @@ opts2args(EnvStartOpts) -> end, EHs), [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), [{event_handler_init,lists:reverse(StrsR)}]; + ({ct_hooks,[]}) -> + []; + ({ct_hooks,CTHs}) when is_list(CTHs) -> + io:format(user,"ct_hooks: ~p",[CTHs]), + Strs = lists:flatmap( + fun({CTH,Arg}) -> + [atom_to_list(CTH), + lists:flatten( + io_lib:format("~p",[Arg])), + "and"]; + (CTH) when is_atom(CTH) -> + [atom_to_list(CTH),"and"] + end,CTHs), + [_LastAnd|StrsR] = lists:reverse(Strs), + io:format(user,"return: ~p",[lists:reverse(StrsR)]), + [{ct_hooks,lists:reverse(StrsR)}]; ({Opt,As=[A|_]}) when is_atom(A) -> [{Opt,[atom_to_list(Atom) || Atom <- As]}]; ({Opt,Strs=[S|_]}) when is_list(S) -> @@ -2263,12 +2332,19 @@ do_trace(Terms) -> dbg:tracer(), dbg:p(self(), [sos,call]), lists:foreach(fun({m,M}) -> - case dbg:tpl(M,[{'_',[],[{return_trace}]}]) of + case dbg:tpl(M,x) of + {error,What} -> exit({error,{tracing_failed,What}}); + _ -> ok + end; + ({me,M}) -> + case dbg:tp(M,[{'_',[],[{exception_trace}, + {message,{caller}}]}]) of {error,What} -> exit({error,{tracing_failed,What}}); _ -> ok end; ({f,M,F}) -> - case dbg:tpl(M,F,[{'_',[],[{return_trace}]}]) of + case dbg:tpl(M,F,[{'_',[],[{exception_trace}, + {message,{caller}}]}]) of {error,What} -> exit({error,{tracing_failed,What}}); _ -> ok end; diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index f5069427a2..d845358bb2 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2010. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -68,7 +68,8 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% Create initial list of {Node,{Run,Skip}} tuples NodeList = lists:map(fun(N) -> {N,{[],[]}} end, list_nodes(TestSpec)), %% Get all Run tests sorted per node basis. - NodeList1 = run_per_node(Run,NodeList), + NodeList1 = run_per_node(Run,NodeList, + TestSpec#testspec.merge_tests), %% Get all Skip entries sorted per node basis. NodeList2 = skip_per_node(Skip,NodeList1), %% Change representation. @@ -89,11 +90,17 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% run_per_node/2 takes the Run list as input and returns a list %% of {Node,RunPerNode,[]} tuples where the tests have been sorted %% on a per node basis. -run_per_node([{{Node,Dir},Test}|Ts],Result) -> +run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) -> {value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result), - Run1 = merge_tests(Dir,Test,Run), - run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result)); -run_per_node([],Result) -> + Run1 = case MergeTests of + false -> + append({Dir, Test}, Run); + true -> + merge_tests(Dir,Test,Run) + end, + run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result), + MergeTests); +run_per_node([],Result,_) -> Result. merge_tests(Dir,Test={all,_},TestDirs) -> @@ -281,6 +288,8 @@ collect_tests(Terms,TestSpec,Relaxed) -> {Terms2, TestSpec3} = filter_init_terms(Terms, [], TestSpec2), add_tests(Terms2,TestSpec3). +get_global([{merge_tests, Bool} | Ts], Spec) -> + get_global(Ts,Spec#testspec{ merge_tests = Bool }); get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) -> get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]}); get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) -> @@ -394,8 +403,6 @@ filter_init_terms([Term|Ts], NewTerms, Spec)-> filter_init_terms([], NewTerms, Spec)-> {lists:reverse(NewTerms), Spec}. -add_option([], _, List, _)-> - List; add_option({Key, Value}, Node, List, WarnIfExists) when is_list(Value)-> OldOptions = case lists:keyfind(Node, 1, List) of {Node, Options}-> @@ -625,6 +632,20 @@ add_tests([{event_handler,Node,H,Args}|Ts],Spec) when is_atom(H) -> Node1 = ref2node(Node,Spec#testspec.nodes), add_tests(Ts,Spec#testspec{event_handler=[{Node1,H,Args}|EvHs]}); +%% --- ct_hooks -- +add_tests([{ct_hooks, all_nodes, Hooks} | Ts], Spec) -> + Tests = [{ct_hooks,N,Hooks} || N <- list_nodes(Spec)], + add_tests(Tests ++ Ts, Spec); +add_tests([{ct_hooks, Node, [Hook|Hooks]}|Ts], Spec) -> + SuiteCbs = Spec#testspec.ct_hooks, + Node1 = ref2node(Node,Spec#testspec.nodes), + add_tests([{ct_hooks, Node, Hooks} | Ts], + Spec#testspec{ct_hooks = [{Node1,Hook} | SuiteCbs]}); +add_tests([{ct_hooks, _Node, []}|Ts], Spec) -> + add_tests(Ts, Spec); +add_tests([{ct_hooks, Hooks}|Ts], Spec) -> + add_tests([{ct_hooks, all_nodes, Hooks}|Ts], Spec); + %% --- include --- add_tests([{include,all_nodes,InclDirs}|Ts],Spec) -> Tests = lists:map(fun(N) -> {include,N,InclDirs} end, list_nodes(Spec)), @@ -656,7 +677,7 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_suites(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Ss,Tests), + Ss,Tests, Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- groups --- @@ -682,13 +703,15 @@ add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,all,Tests), + Suite,Gs,all,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,TCs,Tests), + Suite,Gs,TCs,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- cases --- @@ -703,7 +726,7 @@ add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Cs,Tests), + Suite,Cs,Tests, Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_suites --- @@ -718,7 +741,8 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_suites(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Ss,Cmt,Tests), + Ss,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_groups --- @@ -740,13 +764,15 @@ add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,all,Cmt,Tests), + Suite,Gs,all,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Gs,TCs,Cmt,Tests), + Suite,Gs,TCs,Cmt,Tests, + Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- skip_cases --- @@ -761,7 +787,7 @@ add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes), ref2dir(Dir,Spec#testspec.alias), - Suite,Cs,Cmt,Tests), + Suite,Cs,Cmt,Tests,Spec#testspec.merge_tests), add_tests(Ts,Spec#testspec{tests=Tests1}); %% --- handled/errors --- @@ -771,6 +797,9 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled add_tests([{node,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); +add_tests([{merge_tests, _} | Ts], Spec) -> % handled + add_tests(Ts,Spec); + %% check if it's a CT term that has bad format or if the user seems to %% have added something of his/her own, which we'll let pass if relaxed %% mode is enabled. @@ -823,17 +852,22 @@ separate([],_,_,_) -> %% {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]} %% GrOrCase = {GroupName,[Case1,Case2,...]} | Case -insert_suites(Node,Dir,[S|Ss],Tests) -> - Tests1 = insert_cases(Node,Dir,S,all,Tests), - insert_suites(Node,Dir,Ss,Tests1); -insert_suites(_Node,_Dir,[],Tests) -> +insert_suites(Node,Dir,[S|Ss],Tests, MergeTests) -> + Tests1 = insert_cases(Node,Dir,S,all,Tests,MergeTests), + insert_suites(Node,Dir,Ss,Tests1,MergeTests); +insert_suites(_Node,_Dir,[],Tests,_MergeTests) -> Tests; -insert_suites(Node,Dir,S,Tests) -> - insert_suites(Node,Dir,[S],Tests). +insert_suites(Node,Dir,S,Tests,MergeTests) -> + insert_suites(Node,Dir,[S],Tests,MergeTests). -insert_groups(Node,Dir,Suite,Group,Cases,Tests) when is_atom(Group) -> - insert_groups(Node,Dir,Suite,[Group],Cases,Tests); -insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when +insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests) + when is_atom(Group) -> + insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Groups1 = [{Gr,Cases} || Gr <- Groups], + append({{Node,Dir},[{Suite,Groups1}]},Tests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> @@ -847,9 +881,10 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when Groups1 = [{Gr,Cases} || Gr <- Groups], insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests) end; -insert_groups(Node,Dir,Suite,Groups,Case,Tests) when is_atom(Case) -> +insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests) + when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, - insert_groups(Node,Dir,Suite,Groups,Cases,Tests). + insert_groups(Node,Dir,Suite,Groups,Cases,Tests, MergeTests). insert_groups1(_Suite,_Groups,all) -> all; @@ -879,7 +914,9 @@ insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) -> insert_groups2([],GrAndCases) -> GrAndCases. -insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> +insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) -> + append({{Node,Dir},[{Suite,Cases}]},Tests); +insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> Tests; @@ -889,8 +926,8 @@ insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> false -> insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests) end; -insert_cases(Node,Dir,Suite,Case,Tests) when is_atom(Case) -> - insert_cases(Node,Dir,Suite,[Case],Tests). +insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) -> + insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests). insert_cases1(_Suite,_Cases,all) -> all; @@ -905,22 +942,28 @@ insert_cases1(Suite,Cases,Suites0) -> insert_in_order({Suite,Cases},Suites0) end. -skip_suites(Node,Dir,[S|Ss],Cmt,Tests) -> - Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests), - skip_suites(Node,Dir,Ss,Cmt,Tests1); -skip_suites(_Node,_Dir,[],_Cmt,Tests) -> +skip_suites(Node,Dir,[S|Ss],Cmt,Tests,MergeTests) -> + Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests,MergeTests), + skip_suites(Node,Dir,Ss,Cmt,Tests1,MergeTests); +skip_suites(_Node,_Dir,[],_Cmt,Tests,_MergeTests) -> Tests; -skip_suites(Node,Dir,S,Cmt,Tests) -> - skip_suites(Node,Dir,[S],Cmt,Tests). - -skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); -skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case), - Case =/= all -> - skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when +skip_suites(Node,Dir,S,Cmt,Tests,MergeTests) -> + skip_suites(Node,Dir,[S],Cmt,Tests,MergeTests). + +skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests,MergeTests) + when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests,MergeTests) + when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) + when is_atom(Case),Case =/= all -> + skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests,MergeTests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,[]), + append({{Node,Dir},Suites1},Tests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of @@ -931,9 +974,10 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when end, Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites), insert_in_order({{Node,Dir},Suites1},Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) + when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, - skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests). + skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,MergeTests). skip_groups1(Suite,Groups,Cmt,Suites0) -> SkipGroups = lists:map(fun(Group) -> @@ -947,7 +991,10 @@ skip_groups1(Suite,Groups,Cmt,Suites0) -> insert_in_order({Suite,SkipGroups},Suites0) end. -skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> +skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) -> + Suites1 = skip_cases1(Suite,Cases,Cmt,[]), + append({{Node,Dir},Suites1},Tests); +skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},Suites0}} -> @@ -957,8 +1004,8 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> end, Suites1 = skip_cases1(Suite,Cases,Cmt,Suites), insert_in_order({{Node,Dir},Suites1},Tests); -skip_cases(Node,Dir,Suite,Case,Cmt,Tests) when is_atom(Case) -> - skip_cases(Node,Dir,Suite,[Case],Cmt,Tests). +skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) -> + skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests). skip_cases1(Suite,Cases,Cmt,Suites0) -> SkipCases = lists:map(fun(C) -> @@ -972,6 +1019,9 @@ skip_cases1(Suite,Cases,Cmt,Suites0) -> insert_in_order({Suite,SkipCases},Suites0) end. +append(Elem, List) -> + List ++ [Elem]. + insert_in_order([E|Es],List) -> List1 = insert_elem(E,List,[]), insert_in_order(Es,List1); @@ -1044,6 +1094,7 @@ valid_terms() -> {userconfig,2}, {userconfig,3}, {alias,3}, + {merge_tests,1}, {logdir,2}, {logdir,3}, {label,2}, @@ -1051,6 +1102,8 @@ valid_terms() -> {event_handler,2}, {event_handler,3}, {event_handler,4}, + {ct_hooks,2}, + {ct_hooks,3}, {multiply_timetraps,2}, {multiply_timetraps,3}, {scale_timetraps,2}, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index b5ab4cbb6e..115207beed 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% Copyright Ericsson AB 2003-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -32,7 +32,9 @@ -export([close_connections/0]). --export([save_suite_data/3, save_suite_data/2, read_suite_data/1, +-export([save_suite_data/3, save_suite_data/2, + save_suite_data_async/3, save_suite_data_async/2, + read_suite_data/1, delete_suite_data/0, delete_suite_data/1, match_delete_suite_data/1, delete_testdata/0, delete_testdata/1, set_testdata/1, get_testdata/1, update_testdata/2]). @@ -159,6 +161,17 @@ do_start(Parent,Mode,LogDir) -> ok end, {StartTime,TestLogDir} = ct_logs:init(Mode), + + %% Initiate ct_hooks + case catch ct_hooks:init(Opts) of + ok -> + ok; + {_,CTHReason} -> + ct_logs:tc_print('Suite Callback',CTHReason,[]), + Parent ! {self(), CTHReason}, + self() ! {{stop,normal},{self(),make_ref()}} + end, + ct_event:notify(#event{name=test_start, node=node(), data={StartTime, @@ -182,12 +195,19 @@ read_opts() -> {error,{bad_installation,Error}} end. + save_suite_data(Key, Value) -> call({save_suite_data, {Key, undefined, Value}}). save_suite_data(Key, Name, Value) -> call({save_suite_data, {Key, Name, Value}}). +save_suite_data_async(Key, Value) -> + save_suite_data_async(Key, undefined, Value). + +save_suite_data_async(Key, Name, Value) -> + cast({save_suite_data, {Key, Name, Value}}). + read_suite_data(Key) -> call({read_suite_data, Key}). @@ -268,6 +288,9 @@ loop(Mode,TestData,StartDir) -> TestData1 = lists:keydelete(Key,1,TestData), return(From,ok), loop(Mode,[New|TestData1],StartDir); + {{get_testdata, all}, From} -> + return(From, TestData), + loop(From, TestData, StartDir); {{get_testdata,Key},From} -> case lists:keysearch(Key,1,TestData) of {value,{Key,Val}} -> @@ -299,6 +322,10 @@ loop(Mode,TestData,StartDir) -> ct_event:sync_notify(#event{name=test_done, node=node(), data=Time}), + Callbacks = ets:lookup_element(?suite_table, + ct_hooks, + #suite_data.value), + ct_hooks:terminate(Callbacks), close_connections(ets:tab2list(?conn_table)), ets:delete(?conn_table), ets:delete(?board_table), @@ -308,6 +335,9 @@ loop(Mode,TestData,StartDir) -> ct_config:stop(), file:set_cwd(StartDir), return(From,ok); + {Ref, _Msg} when is_reference(Ref) -> + %% This clause is used when doing cast operations. + loop(Mode,TestData,StartDir); {get_mode,From} -> return(From,Mode), loop(Mode,TestData,StartDir); @@ -713,6 +743,9 @@ call(Msg) -> return({To,Ref},Result) -> To ! {Ref, Result}. +cast(Msg) -> + ct_util_server ! {Msg, {ct_util_server, make_ref()}}. + seconds(T) -> test_server:seconds(T). diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index ee973f6220..556f88c84d 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% Copyright Ericsson AB 2003-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -36,11 +36,13 @@ config=[], userconfig=[], event_handler=[], + ct_hooks=[], include=[], multiply_timetraps=[], scale_timetraps=[], alias=[], - tests=[]}). + tests=[], + merge_tests = true }). -record(cover, {app=none, level=details, diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index fed18d5a31..b7b099069c 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2010. All Rights Reserved. +# Copyright Ericsson AB 2008-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -40,7 +40,8 @@ MODULES= \ ct_test_server_if_1_SUITE \ ct_config_SUITE \ ct_master_SUITE \ - ct_misc_1_SUITE + ct_misc_1_SUITE \ + ct_hooks_SUITE ERL_FILES= $(MODULES:%=%.erl) diff --git a/lib/common_test/test/common_test.spec b/lib/common_test/test/common_test.spec index 7619a75b31..8755b08117 100644 --- a/lib/common_test/test/common_test.spec +++ b/lib/common_test/test/common_test.spec @@ -1 +1 @@ -{topcase, {dir, "../common_test"}}. +{suites,"../common_test_test",all}.
\ No newline at end of file diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index fc15abc5bc..b6b50f33e0 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -64,19 +64,22 @@ end_per_testcase(install_config = TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [require, install_config, userconfig_static, + userconfig_dynamic, testspec_legacy, testspec_static, + testspec_dynamic]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [ - require, - install_config, - userconfig_static, - userconfig_dynamic, - testspec_legacy, - testspec_static, - testspec_dynamic - ]. %%-------------------------------------------------------------------- %% TEST CASES @@ -199,74 +202,74 @@ events_to_check(Test, N) -> expected_events(Test) ++ events_to_check(Test, N-1). expected_events(config_static_SUITE)-> -[ - {?eh,start_logging,{'DEF','RUNDIR'}}, - {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,8}}, - {?eh,tc_start,{config_static_SUITE,init_per_suite}}, - {?eh,tc_done,{config_static_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{config_static_SUITE,test_get_config_simple}}, - {?eh,tc_done,{config_static_SUITE,test_get_config_simple,ok}}, - {?eh,test_stats,{1,0,{0,0}}}, - {?eh,tc_start,{config_static_SUITE,test_get_config_nested}}, - {?eh,tc_done,{config_static_SUITE,test_get_config_nested,ok}}, - {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{config_static_SUITE,test_default_suitewide}}, - {?eh,tc_done,{config_static_SUITE,test_default_suitewide,ok}}, - {?eh,test_stats,{3,0,{0,0}}}, - {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use1}}, - {?eh,tc_done, - {config_static_SUITE,test_config_name_already_in_use1,{skipped,{config_name_already_in_use,[x1]}}}}, - {?eh,test_stats,{3,0,{1,0}}}, - {?eh,tc_start,{config_static_SUITE,test_default_tclocal}}, - {?eh,tc_done,{config_static_SUITE,test_default_tclocal,ok}}, - {?eh,test_stats,{4,0,{1,0}}}, - {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}}, - {?eh,tc_done, - {config_static_SUITE,test_config_name_already_in_use2, - {skipped,{config_name_already_in_use,[x1,alias]}}}}, - {?eh,test_stats,{4,0,{2,0}}}, - {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}}, - {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}}, - {?eh,test_stats,{5,0,{2,0}}}, - {?eh,tc_start,{config_static_SUITE,test_get_config_undefined}}, - {?eh,tc_done,{config_static_SUITE,test_get_config_undefined,ok}}, - {?eh,test_stats,{6,0,{2,0}}}, - {?eh,tc_start,{config_static_SUITE,end_per_suite}}, - {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, - {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]} -]; + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,8}}, + {?eh,tc_start,{config_static_SUITE,init_per_suite}}, + {?eh,tc_done,{config_static_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_simple}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_simple,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_nested}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_nested,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_default_suitewide}}, + {?eh,tc_done,{config_static_SUITE,test_default_suitewide,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use1}}, + {?eh,tc_done, + {config_static_SUITE,test_config_name_already_in_use1,{skipped,{config_name_already_in_use,[x1]}}}}, + {?eh,test_stats,{3,0,{1,0}}}, + {?eh,tc_start,{config_static_SUITE,test_default_tclocal}}, + {?eh,tc_done,{config_static_SUITE,test_default_tclocal,ok}}, + {?eh,test_stats,{4,0,{1,0}}}, + {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}}, + {?eh,tc_done, + {config_static_SUITE,test_config_name_already_in_use2, + {skipped,{config_name_already_in_use,[x1,alias]}}}}, + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}}, + {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}}, + {?eh,test_stats,{5,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_undefined}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_undefined,ok}}, + {?eh,test_stats,{6,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,end_per_suite}}, + {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; expected_events(config_dynamic_SUITE)-> -[ - {?eh,start_logging,{'DEF','RUNDIR'}}, - {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,5}}, - {?eh,tc_start,{config_dynamic_SUITE,init_per_suite}}, - {?eh,tc_done,{config_dynamic_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{config_dynamic_SUITE,test_get_known_variable}}, - {?eh,tc_done, - {config_dynamic_SUITE,test_get_known_variable,ok}}, - {?eh,test_stats,{1,0,{0,0}}}, - {?eh,tc_start,{config_dynamic_SUITE,test_localtime_update}}, - {?eh,tc_done,{config_dynamic_SUITE,test_localtime_update,ok}}, - {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{config_dynamic_SUITE,test_server_pid}}, - {?eh,tc_done,{config_dynamic_SUITE,test_server_pid,ok}}, - {?eh,test_stats,{3,0,{0,0}}}, - {?eh,tc_start, - {config_dynamic_SUITE,test_disappearable_variable}}, - {?eh,tc_done, - {config_dynamic_SUITE,test_disappearable_variable,ok}}, - {?eh,test_stats,{4,0,{0,0}}}, - {?eh,tc_start, - {config_dynamic_SUITE,test_disappearable_variable_alias}}, - {?eh,tc_done, - {config_dynamic_SUITE,test_disappearable_variable_alias,ok}}, - {?eh,test_stats,{5,0,{0,0}}}, - {?eh,tc_start,{config_dynamic_SUITE,end_per_suite}}, - {?eh,tc_done,{config_dynamic_SUITE,end_per_suite,ok}}, - {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]} -]. + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,5}}, + {?eh,tc_start,{config_dynamic_SUITE,init_per_suite}}, + {?eh,tc_done,{config_dynamic_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_dynamic_SUITE,test_get_known_variable}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_get_known_variable,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,test_localtime_update}}, + {?eh,tc_done,{config_dynamic_SUITE,test_localtime_update,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,test_server_pid}}, + {?eh,tc_done,{config_dynamic_SUITE,test_server_pid,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start, + {config_dynamic_SUITE,test_disappearable_variable}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_disappearable_variable,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {config_dynamic_SUITE,test_disappearable_variable_alias}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_disappearable_variable_alias,ok}}, + {?eh,test_stats,{5,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,end_per_suite}}, + {?eh,tc_done,{config_dynamic_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 2fa031b884..ad6cf1ba8f 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,19 +56,22 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [cfg_error, lib_error, no_compile, timetrap_end_conf, + timetrap_normal, timetrap_extended]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + -all(suite) -> - [ - cfg_error, - lib_error, - no_compile, - timetrap_end_conf, - timetrap_normal, - timetrap_extended - ]. - %%-------------------------------------------------------------------- %% TEST CASES @@ -122,7 +125,7 @@ lib_error(Config) when is_list(Config) -> TestEvents = events_to_check(lib_error), ok = ct_test_support:verify_events(TestEvents, Events, Config). - + %%%----------------------------------------------------------------- %%% @@ -140,7 +143,7 @@ no_compile(Config) when is_list(Config) -> TestEvents = events_to_check(no_compile), ok = ct_test_support:verify_events(TestEvents, Events, Config). - + %%%----------------------------------------------------------------- %%% timetrap_end_conf(Config) when is_list(Config) -> @@ -214,8 +217,8 @@ setup(Test, Config) -> reformat(Events, EH) -> ct_test_support:reformat(Events, EH). -%reformat(Events, _EH) -> -% Events. + %reformat(Events, _EH) -> + % Events. %%%----------------------------------------------------------------- %%% TEST EVENTS @@ -248,7 +251,7 @@ test_events(cfg_error) -> {?eh,test_stats,{0,0,{0,2}}}, {?eh,tc_auto_skip, {cfg_error_1_SUITE,end_per_suite,{failed,{cfg_error_1_SUITE,init_per_suite, - {'EXIT',init_per_suite_fails}}}}}, + {'EXIT',init_per_suite_fails}}}}}, {?eh,tc_start,{cfg_error_2_SUITE,init_per_suite}}, {?eh,tc_done, @@ -378,12 +381,12 @@ test_events(cfg_error) -> {?eh,tc_auto_skip, {cfg_error_8_SUITE,tc1, {failed,{cfg_error_8_SUITE,init_per_group, - {'EXIT',{init_per_group_fails,g1}}}}}}, + {'EXIT',{init_per_group_fails,g1}}}}}}, {?eh,test_stats,{4,0,{0,11}}}, {?eh,tc_auto_skip, {cfg_error_8_SUITE,end_per_group, {failed,{cfg_error_8_SUITE,init_per_group, - {'EXIT',{init_per_group_fails,g1}}}}}}], + {'EXIT',{init_per_group_fails,g1}}}}}}], [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}}, {?eh,tc_done,{cfg_error_8_SUITE, @@ -396,7 +399,7 @@ test_events(cfg_error) -> {?eh,tc_auto_skip,{cfg_error_8_SUITE,end_per_group, {failed,{cfg_error_8_SUITE,init_per_group, {timetrap_timeout,2000}}}}}], - + [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done, {cfg_error_8_SUITE,{init_per_group,g3,[]}, @@ -436,7 +439,7 @@ test_events(cfg_error) -> {?eh,test_stats,{5,0,{0,13}}}, {?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g4,[]}}}, {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g4,[]},ok}}], - + [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g5,[]}}}, {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_start,{cfg_error_8_SUITE,tc1}}, diff --git a/lib/common_test/test/ct_event_handler_SUITE.erl b/lib/common_test/test/ct_event_handler_SUITE.erl index 00a4c4ded3..5ef04c0e75 100644 --- a/lib/common_test/test/ct_event_handler_SUITE.erl +++ b/lib/common_test/test/ct_event_handler_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). %-include_lib("common_test/include/ct_event.hrl"). @@ -56,12 +56,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - []; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> +all() -> [start_stop, results]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_groups_test_1_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE.erl index 64d61fc104..7775d8a55d 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,12 +56,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - ["Run smoke tests of Common Test."]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [groups_suite_1, groups_suite_2, groups_suites_1, + groups_dir_1, groups_dirs_1]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [groups_suite_1, groups_suite_2, - groups_suites_1, groups_dir_1, groups_dirs_1]. %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index c4371501b3..2ae63f4f99 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,12 +56,21 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - ["Run smoke tests of Common Test."]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> +all() -> [missing_conf, repeat_1]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl new file mode 100644 index 0000000000..64f4e277ff --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -0,0 +1,1021 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_error_SUITE +%%% +%%% Description: +%%% Test various errors in Common Test suites. +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_hooks_SUITE). + +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir,"cth/tests/"), + CTHs = filelib:wildcard(filename:join(TestDir,"*_cth.erl")), + io:format("CTHs: ~p",[CTHs]), + [io:format("Compiling ~p: ~p", + [FileName,compile:file(FileName,[{outdir,TestDir},debug_info])]) || + FileName <- CTHs], + ct_test_support:init_per_suite([{path_dirs,[TestDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{seconds,20}}]. + +all() -> + all(suite). + +all(suite) -> + lists:reverse( + [ + one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, + faulty_cth_exit_in_init, faulty_cth_exit_in_id, + faulty_cth_exit_in_init_scope_suite, minimal_cth, + minimal_and_maximal_cth, faulty_cth_undef, + scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, + scope_per_suite_state_cth, scope_per_group_state_cth, + scope_suite_state_cth, + fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, + skip_post_suite_cth, recover_post_suite_cth, update_config_cth, + state_update_cth, options_cth, same_id_cth, + fail_n_skip_with_minimal_cth + ] + ) + . + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +one_cth(Config) when is_list(Config) -> + do_test(one_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth], Config). + +two_cth(Config) when is_list(Config) -> + do_test(two_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth,empty_cth], + Config). + +faulty_cth_no_init(Config) when is_list(Config) -> + do_test(faulty_cth_no_init, "ct_cth_empty_SUITE.erl",[askjhdkljashdkaj], + Config,{error,"Failed to start CTH, see the " + "CT Log for details"}). + +faulty_cth_id_no_init(Config) when is_list(Config) -> + do_test(faulty_cth_id_no_init, "ct_cth_empty_SUITE.erl",[id_no_init_cth], + Config,{error,"Failed to start CTH, see the " + "CT Log for details"}). + +minimal_cth(Config) when is_list(Config) -> + do_test(minimal_cth, "ct_cth_empty_SUITE.erl",[minimal_cth],Config). + +minimal_and_maximal_cth(Config) when is_list(Config) -> + do_test(minimal_and_maximal_cth, "ct_cth_empty_SUITE.erl", + [minimal_cth, empty_cth],Config). + +faulty_cth_undef(Config) when is_list(Config) -> + do_test(faulty_cth_undef, "ct_cth_empty_SUITE.erl", + [undef_cth],Config). + +faulty_cth_exit_in_init_scope_suite(Config) when is_list(Config) -> + do_test(faulty_cth_exit_in_init_scope_suite, + "ct_exit_in_init_scope_suite_cth_SUITE.erl", + [],Config). + +faulty_cth_exit_in_init(Config) when is_list(Config) -> + do_test(faulty_cth_exit_in_init, "ct_cth_empty_SUITE.erl", + [crash_init_cth], Config, + {error,"Failed to start CTH, see the " + "CT Log for details"}). + +faulty_cth_exit_in_id(Config) when is_list(Config) -> + do_test(faulty_cth_exit_in_id, "ct_cth_empty_SUITE.erl", + [crash_id_cth], Config, + {error,"Failed to start CTH, see the " + "CT Log for details"}). + +scope_per_suite_cth(Config) when is_list(Config) -> + do_test(scope_per_suite_cth, "ct_scope_per_suite_cth_SUITE.erl", + [],Config). + +scope_suite_cth(Config) when is_list(Config) -> + do_test(scope_suite_cth, "ct_scope_suite_cth_SUITE.erl", + [],Config). + +scope_per_group_cth(Config) when is_list(Config) -> + do_test(scope_per_group_cth, "ct_scope_per_group_cth_SUITE.erl", + [],Config). + +scope_per_suite_state_cth(Config) when is_list(Config) -> + do_test(scope_per_suite_state_cth, "ct_scope_per_suite_state_cth_SUITE.erl", + [],Config). + +scope_suite_state_cth(Config) when is_list(Config) -> + do_test(scope_suite_state_cth, "ct_scope_suite_state_cth_SUITE.erl", + [],Config). + +scope_per_group_state_cth(Config) when is_list(Config) -> + do_test(scope_per_group_state_cth, "ct_scope_per_group_state_cth_SUITE.erl", + [],Config). + +fail_pre_suite_cth(Config) when is_list(Config) -> + do_test(fail_pre_suite_cth, "ct_cth_empty_SUITE.erl", + [fail_pre_suite_cth],Config). + +fail_post_suite_cth(Config) when is_list(Config) -> + do_test(fail_post_suite_cth, "ct_cth_empty_SUITE.erl", + [fail_post_suite_cth],Config). + +skip_pre_suite_cth(Config) when is_list(Config) -> + do_test(skip_pre_suite_cth, "ct_cth_empty_SUITE.erl", + [skip_pre_suite_cth],Config). + +skip_post_suite_cth(Config) when is_list(Config) -> + do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl", + [skip_post_suite_cth],Config). + +recover_post_suite_cth(Config) when is_list(Config) -> + do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl", + [recover_post_suite_cth],Config). + +update_config_cth(Config) when is_list(Config) -> + do_test(update_config_cth, "ct_update_config_SUITE.erl", + [update_config_cth],Config). + +state_update_cth(Config) when is_list(Config) -> + do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl", + [state_update_cth,state_update_cth],Config). + +options_cth(Config) when is_list(Config) -> + do_test(options_cth, "ct_cth_empty_SUITE.erl", + [{empty_cth,[test]}],Config). + +same_id_cth(Config) when is_list(Config) -> + do_test(same_id_cth, "ct_cth_empty_SUITE.erl", + [same_id_cth,same_id_cth],Config). + +fail_n_skip_with_minimal_cth(Config) when is_list(Config) -> + do_test(fail_n_skip_with_minimal_cth, "ct_cth_fail_one_skip_one_SUITE.erl", + [minimal_terminate_cth],Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, SWC, CTHs, Config) -> + do_test(Tag, SWC, CTHs, Config, ok). +do_test(Tag, SWC, CTHs, Config, {error,_} = Res) -> + do_test(Tag, SWC, CTHs, Config, Res, 1); +do_test(Tag, SWC, CTHs, Config, Res) -> + do_test(Tag, SWC, CTHs, Config, Res, 2). + +do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> + + DataDir = ?config(data_dir, Config), + Suites = filelib:wildcard( + filename:join([DataDir,"cth/tests",SuiteWildCard])), + {Opts,ERPid} = setup([{suite,Suites}, + {ct_hooks,CTHs},{label,Tag}], Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). +%reformat(Events, _EH) -> +% Events. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(one_empty_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(two_empty_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_no_init) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_id_no_init) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {negative,{?eh,tc_start,'_'}, + {?eh,test_done,{'DEF','STOP_TIME'}}}, + {?eh,stop_logging,[]} + ]; + +test_events(minimal_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {negative,{?eh,cth,{'_',id,['_',[]]}}, + {?eh,cth,{'_',init,['_',[]]}}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(minimal_and_maximal_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {negative,{?eh,cth,{'_',id,['_',[]]}}, + {?eh,cth,{'_',init,['_',[]]}}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_undef) -> + FailReasonStr = "undef_cth:pre_init_per_suite/3 CTH call failed", + FailReason = {ct_cth_empty_SUITE,init_per_suite, + {failed,FailReasonStr}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, + {failed, {error,FailReasonStr}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, + {failed, FailReason}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,end_per_suite, + {failed, FailReason}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(faulty_cth_exit_in_init_scope_suite) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',init_per_suite}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,tc_done, + {ct_exit_in_init_scope_suite_cth_SUITE,init_per_suite, + {failed, + {error, + "Failed to start CTH, see the CT Log for details"}}}}, + {?eh,tc_auto_skip, + {ct_exit_in_init_scope_suite_cth_SUITE,test_case, + {failed, + {ct_exit_in_init_scope_suite_cth_SUITE,init_per_suite, + {failed, + "Failed to start CTH, see the CT Log for details"}}}}}, + {?eh,tc_auto_skip, + {ct_exit_in_init_scope_suite_cth_SUITE,end_per_suite, + {failed, + {ct_exit_in_init_scope_suite_cth_SUITE,init_per_suite, + {failed, + "Failed to start CTH, see the CT Log for details"}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(faulty_cth_exit_in_init) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}]; + +test_events(faulty_cth_exit_in_id) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {negative, {?eh,tc_start,'_'}, + {?eh,test_done,{'DEF','STOP_TIME'}}}, + {?eh,stop_logging,[]}]; + +test_events(scope_per_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_per_suite_cth_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [ct_scope_per_suite_cth_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_per_suite_cth_SUITE,'$proplist','_',[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_suite_cth_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_suite_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_suite_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist','_',[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,tc_done,{ct_scope_suite_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_per_group_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}], + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_per_suite_state_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[test]]}}, + {?eh,cth,{'_',init,['_',[test]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_per_suite_state_cth_SUITE,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [ct_scope_per_suite_state_cth_SUITE,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_per_suite_state_cth_SUITE,'$proplist','_',[test]]}}, + {?eh,cth,{'_',terminate,[[test]]}}, + {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_suite_state_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,init_per_suite}}, + {?eh,cth,{'_',id,[[test]]}}, + {?eh,cth,{'_',init,['_',[test]]}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist',[test]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist','_',[test]]}}, + {?eh,cth,{'_',terminate,[[test]]}}, + {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(scope_per_group_state_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]}}}, + {?eh,cth,{'_',id,[[test]]}}, + {?eh,cth,{'_',init,['_',[test]]}}, + {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}}, + {?eh,cth,{'_',terminate,[[test]]}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}], + + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(fail_pre_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + + + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist', + {fail,"Test failure"},[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, + {failed, {error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail, + [init_per_suite,{failed,"Test failure"},[]]}}, + + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, + {failed,{ct_cth_empty_SUITE,init_per_suite, + {failed,"Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip, + [test_case, {tc_auto_skip, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}},[]]}}, + + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip, + [end_per_suite, {tc_auto_skip, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}},[]]}}, + + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth, {'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(fail_post_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, + {failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,[init_per_suite, {failed,"Test failure"}, []]}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, + {failed,{ct_cth_empty_SUITE,init_per_suite, + {failed,"Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}}, + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, + {failed, {ct_cth_empty_SUITE, init_per_suite, + {failed, "Test failure"}}}}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,'_'},[]]}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth, {'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(skip_pre_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist',{skip,"Test skip"},[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, + {?eh,cth,{'_',on_tc_skip, + [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth, {'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(skip_post_suite_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, + {?eh,cth,{'_',on_tc_skip, + [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(recover_post_suite_cth) -> + Suite = ct_cth_fail_per_suite_SUITE, + [ + {?eh,start_logging,'_'}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{Suite,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[Suite,'$proplist','$proplist']}}, + {?eh,cth,{'_',post_init_per_suite,[Suite,contains([tc_status]), + {'EXIT',{'_','_'}},[]]}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase, + [test_case, not_contains([tc_status]),[]]}}, + {?eh,cth,{'_',post_end_per_testcase, + [test_case, contains([tc_status]),'_',[]]}}, + {?eh,tc_done,{Suite,test_case,ok}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [Suite,not_contains([tc_status]),[]]}}, + {?eh,cth,{'_',post_end_per_suite, + [Suite,not_contains([tc_status]),'_',[]]}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(update_config_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + + {?eh,tc_start,{ct_update_config_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite, + [ct_update_config_SUITE,contains([]),[]]}}, + {?eh,cth,{'_',post_init_per_suite, + [ct_update_config_SUITE, + '$proplist', + contains( + [init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,tc_done,{ct_update_config_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_init_per_group, + [group1,contains( + [post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_init_per_group, + [group1, + contains( + [post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + contains( + [init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,tc_done,{ct_update_config_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_update_config_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase, + [test_case,contains( + [post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_end_per_testcase, + [test_case,contains( + [init_per_testcase, + pre_init_per_testcase, + post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + ok,[]]}}, + {?eh,tc_done,{ct_update_config_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group, + [group1,contains( + [post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_end_per_group, + [group1, + contains( + [pre_end_per_group, + post_init_per_group, + init_per_group, + pre_init_per_group, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + ok,[]]}}, + {?eh,tc_done,{ct_update_config_SUITE,{end_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_update_config_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite, + [ct_update_config_SUITE,contains( + [post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + []]}}, + {?eh,cth,{'_',post_end_per_suite, + [ct_update_config_SUITE,contains( + [pre_end_per_suite, + post_init_per_suite, + init_per_suite, + pre_init_per_suite]), + '_',[]]}}, + {?eh,tc_done,{ct_update_config_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[contains( + [post_end_per_suite, + pre_end_per_suite, + post_init_per_suite, + init_per_suite, + pre_init_per_suite])]}}, + {?eh,stop_logging,[]} + ]; + +test_events(state_update_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',init_per_suite}}, + + {?eh,tc_done,{'_',end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[contains( + [post_end_per_suite,pre_end_per_suite, + post_end_per_group,pre_end_per_group, + {not_in_order, + [post_end_per_testcase,pre_init_per_testcase, + on_tc_skip,post_end_per_testcase, + pre_init_per_testcase,on_tc_fail, + post_end_per_testcase,pre_init_per_testcase] + }, + post_init_per_group,pre_init_per_group, + post_init_per_suite,pre_init_per_suite, + init])]}}, + {?eh,cth,{'_',terminate,[contains( + [post_end_per_suite,pre_end_per_suite, + post_end_per_group,pre_end_per_group, + {not_in_order, + [post_end_per_testcase,pre_init_per_testcase, + on_tc_skip,post_end_per_testcase, + pre_init_per_testcase,on_tc_fail, + post_end_per_testcase,pre_init_per_testcase] + }, + post_init_per_group,pre_init_per_group, + post_init_per_suite,pre_init_per_suite, + init] + )]}}, + {?eh,stop_logging,[]} + ]; + +test_events(options_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{empty_cth,init,['_',[test]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_cth_empty_SUITE,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[test]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_cth_empty_SUITE,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[test]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[test]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(same_id_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,[same_id_cth,[]]}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {negative, + {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}}, + {negative, + {?eh,cth,{'_',post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {negative, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}}, + {negative, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}}, + + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {negative, + {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}}, + {negative, + {?eh,cth,{'_',post_end_per_suite, + [ct_cth_empty_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(fail_n_skip_with_minimal_cth) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',init_per_suite}}, + + {?eh,tc_done,{'_',end_per_suite,ok}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(ok) -> + ok. + + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_id_cth.erl new file mode 100644 index 0000000000..b5541f2053 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_id_cth.erl @@ -0,0 +1,34 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(crash_id_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([id/1]).
+
+id(Opts) ->
+ empty_cth:id(Opts),
+ exit(diediedie).
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_init_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_init_cth.erl new file mode 100644 index 0000000000..596b4fade0 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/crash_init_cth.erl @@ -0,0 +1,34 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(crash_init_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([init/2]).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts),
+ exit(diediedie).
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_empty_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_empty_SUITE.erl new file mode 100644 index 0000000000..dcba113eab --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_empty_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_empty_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_one_skip_one_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_one_skip_one_SUITE.erl new file mode 100644 index 0000000000..b2f22d8257 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_one_skip_one_SUITE.erl @@ -0,0 +1,64 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_fail_one_skip_one_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_group(_Group,Config) ->
+ Config.
+
+end_per_group(_Group,_Config) ->
+ ok.
+
+init_per_testcase(test_case2, Config) ->
+ {skip,"skip it"};
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+groups() ->
+ [{group1,[parallel],[{group2,[parallel],[test_case1,test_case2,test_case3]}]}].
+
+all() ->
+ [{group,group1}].
+
+%% Test cases starts here.
+test_case1(Config) ->
+ ok = nok.
+
+test_case2(Config) ->
+ ok.
+
+test_case3(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_per_suite_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_per_suite_SUITE.erl new file mode 100644 index 0000000000..48816523c7 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_fail_per_suite_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_cth_fail_per_suite_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ ok = nok.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_exit_in_init_scope_suite_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_exit_in_init_scope_suite_cth_SUITE.erl new file mode 100644 index 0000000000..6fa77128ab --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_exit_in_init_scope_suite_cth_SUITE.erl @@ -0,0 +1,50 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_exit_in_init_scope_suite_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[crash_init_cth]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_cth_SUITE.erl new file mode 100644 index 0000000000..18af37096a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_cth_SUITE.erl @@ -0,0 +1,56 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_group_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(GroupName, Config) ->
+ [{ct_hooks,[empty_cth]}|Config].
+
+end_per_group(GroupName, Config) ->
+ ok.
+
+all() ->
+ [{group,group1}].
+
+groups() ->
+ [{group1,[],[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_state_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_state_cth_SUITE.erl new file mode 100644 index 0000000000..a34474ebfd --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_group_state_cth_SUITE.erl @@ -0,0 +1,56 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_group_state_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(_GroupName, Config) ->
+ [{ct_hooks,[{empty_cth,[test]}]}|Config].
+
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+all() ->
+ [{group,group1}].
+
+groups() ->
+ [{group1,[],[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_cth_SUITE.erl new file mode 100644 index 0000000000..a3a8f2602f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_cth_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_suite_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ [{ct_hooks,[empty_cth]}|Config].
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_state_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_state_cth_SUITE.erl new file mode 100644 index 0000000000..3f643d6709 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_suite_state_cth_SUITE.erl @@ -0,0 +1,47 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_suite_state_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ [{ct_hooks,[{empty_cth,[test]}]}|Config].
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl new file mode 100644 index 0000000000..1c942937eb --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl @@ -0,0 +1,110 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_per_tc_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+%%--------------------------------------------------------------------
+%% @doc
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Initiation before the whole suite
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%%
+%% @spec init_per_suite(Config) -> Config
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Cleanup after the whole suite
+%%
+%% @spec end_per_suite(Config) -> _
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Case - atom()
+%% Name of the test case that is about to be run.
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Initiation before each test case
+%%
+%% Note: This function is free to add any key/value pairs to the Config
+%% variable, but should NOT alter/remove any existing entries.
+%% Initiation before each test case
+%%
+%% @spec init_per_testcase(TestCase, Config) -> Config
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ [{ct_hooks,[empty_cth]}|Config].
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Case - atom()
+%% Name of the test case that is about to be run.
+%% Config - [tuple()]
+%% A list of key/value pairs, holding the test case configuration.
+%%
+%% Cleanup after each test case
+%%
+%% @spec end_per_testcase(TestCase, Config) -> _
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @doc
+%% TestCases - [Case]
+%% Case - atom()
+%% Name of a test case.
+%%
+%% Returns a list of all test cases in this test suite
+%%
+%% @spec all() -> TestCases
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+%%--------------------------------------------------------------------
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_cth_SUITE.erl new file mode 100644 index 0000000000..482e87a54f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_cth_SUITE.erl @@ -0,0 +1,50 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_suite_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[empty_cth]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_state_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_state_cth_SUITE.erl new file mode 100644 index 0000000000..7b4c9b3fab --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_state_cth_SUITE.erl @@ -0,0 +1,50 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_suite_state_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[{empty_cth,[test]}]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl new file mode 100644 index 0000000000..3c1f5669e8 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_update_config_SUITE.erl @@ -0,0 +1,56 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_update_config_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ [{init_per_suite,now()}|Config].
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ [{init_per_testcase,now()}|Config].
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+init_per_group(GroupName, Config) ->
+ [{init_per_group,now()}|Config].
+
+end_per_group(GroupName, Config) ->
+ ok.
+
+all() ->
+ [{group,group1}].
+
+groups() ->
+ [{group1,[],[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl new file mode 100644 index 0000000000..ebebfd18a9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -0,0 +1,278 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% @doc Common Test Example Suite Callback module.
+%%%
+%%% <p>This module gives an example of a common test CTH (Common Test Hook).
+%%% There are many ways to add a CTH to a test run, you can do it either in
+%%% the command line using -ct_hook, in a test spec using
+%%% {ct_hook,M} or in the suite it self by returning ct_hook
+%%% from either suite/0, init_per_suite/1, init_per_group/2 and
+%%% init_per_testcase/2. The scope of the CTH is determined by where is it
+%%% started. If it is started in the command line or test spec then it will
+%%% be stopped at the end of all tests. If it is started in init_per_suite,
+%%% it will be stopped after end_per_suite and so on. See terminate
+%%% documentation for a table describing the scoping machanics.
+%%%
+%%% All of callbacks except init/1 in a CTH are optional.</p>
+
+-module(empty_cth).
+
+%% CT Hooks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-type proplist() :: list({atom(),term()}).
+-type config() :: proplist().
+-type reason() :: term().
+-type skip_or_fail() :: {skip, reason()} |
+ {auto_skip, reason()} |
+ {fail, reason()} |
+ {'EXIT',reason()}.
+
+-record(state, { id = ?MODULE :: term()}).
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. It should return an state for this CTH.
+-spec init(Id :: term(), Opts :: proplist()) ->
+ State :: #state{}.
+init(Id, Opts) ->
+ gen_event:notify(?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, init, [Id, Opts]}}),
+ Opts.
+
+%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's
+%% return the same ID the seconds CTH is ignored. This function should NOT
+%% have any side effects as it might be called multiple times by common test.
+-spec id(Opts :: proplist()) ->
+ Id :: term().
+id(Opts) ->
+ gen_event:notify(?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, id, [Opts]}}),
+ now().
+
+%% @doc Called before init_per_suite is called. Note that this callback is
+%% only called if the CTH is added before init_per_suite is run (eg. in a test
+%% specification, suite/0 function etc).
+%% You can change the config in the this function.
+-spec pre_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_suite(Suite,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_init_per_suite,
+ [Suite,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after init_per_suite.
+%% you can change the return value in this function.
+-spec post_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_suite(Suite,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_init_per_suite,
+ [Suite,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called before end_per_suite. The config/state can be changed here,
+%% though it will only affect the *end_per_suite function.
+-spec pre_end_per_suite(Suite :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_suite(Suite,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_end_per_suite,
+ [Suite,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after end_per_suite. Note that the config cannot be
+%% changed here, only the status of the suite.
+-spec post_end_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_suite(Suite,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_suite,
+ [Suite,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called before each init_per_group.
+%% You can change the config in this function.
+-spec pre_init_per_group(Group :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_group(Group,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_init_per_group,
+ [Group,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after each init_per_group.
+%% You can change the return value in this function.
+-spec post_init_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_group(Group,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_init_per_group,
+ [Group,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called after each end_per_group. The config/state can be changed here,
+%% though it will only affect the *end_per_group functions.
+-spec pre_end_per_group(Group :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_group(Group,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_end_per_group,
+ [Group,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after each end_per_group. Note that the config cannot be
+%% changed here, only the status of the group.
+-spec post_end_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_group(Group,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_group,
+ [Group,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called before each test case.
+%% You can change the config in this function.
+-spec pre_init_per_testcase(TC :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_testcase(TC,Config,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, pre_init_per_testcase,
+ [TC,Config,State]}}),
+ {Config, State}.
+
+%% @doc Called after each test case. Note that the config cannot be
+%% changed here, only the status of the test case.
+-spec post_end_per_testcase(TC :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_testcase(TC,Config,Return,State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_testcase,
+ [TC,Config,Return,State]}}),
+ {Return, State}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_tc if the suite, group or test case failed.
+%% This function should be used for extra cleanup which might be needed.
+%% It is not possible to modify the config or the status of the test run.
+-spec on_tc_fail(TC :: init_per_suite | end_per_suite |
+ init_per_group | end_per_group | atom(),
+ Reason :: term(), State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_fail(TC, Reason, State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, on_tc_fail,
+ [TC,Reason,State]}}),
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. Test case can be
+%% end_per_suite, init_per_group, end_per_group and the actual test cases.
+-spec on_tc_skip(TC :: end_per_suite |
+ init_per_group | end_per_group | atom(),
+ {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |
+ {tc_user_skip, {skipped, Reason :: term()}},
+ State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_skip(TC, Reason, State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, on_tc_skip,
+ [TC,Reason,State]}}),
+ State.
+
+%% @doc Called when the scope of the CTH is done, this depends on
+%% when the CTH was specified. This translation table describes when this
+%% function is called.
+%%
+%% | Started in | terminate called |
+%% |---------------------|-------------------------|
+%% | command_line | after all tests are run |
+%% | test spec | after all tests are run |
+%% | suite/0 | after SUITE is done |
+%% | init_per_suite/1 | after SUITE is done |
+%% | init_per_group/2 | after group is done |
+%% |-----------------------------------------------|
+%%
+-spec terminate(State :: #state{}) ->
+ term().
+terminate(State) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, terminate, [State]}}),
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl new file mode 100644 index 0000000000..5af9906df0 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl @@ -0,0 +1,72 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fail_post_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {{fail, "Test failure"}, State}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl new file mode 100644 index 0000000000..8227b408cd --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl @@ -0,0 +1,72 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fail_pre_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {{fail, "Test failure"}, State}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/id_no_init_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/id_no_init_cth.erl new file mode 100644 index 0000000000..1e222c1dbf --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/id_no_init_cth.erl @@ -0,0 +1,32 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(id_no_init_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([id/1]).
+
+id(Opts) ->
+ empty_cth:id(Opts).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_cth.erl new file mode 100644 index 0000000000..b87da4e330 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_cth.erl @@ -0,0 +1,33 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(minimal_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([init/2]).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl new file mode 100644 index 0000000000..30721a6b3a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl @@ -0,0 +1,38 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(minimal_terminate_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-export([init/2]).
+-export([terminate/1]).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+terminate(State) ->
+ empty_cth:terminate(State).
+
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl new file mode 100644 index 0000000000..2629448943 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl @@ -0,0 +1,74 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(recover_post_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,{'EXIT',Reason} = Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {lists:keydelete(tc_status,1,Config),State};
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl new file mode 100644 index 0000000000..49b1b9cada --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl @@ -0,0 +1,75 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(same_id_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts),
+ ?MODULE.
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl new file mode 100644 index 0000000000..770fec0a51 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl @@ -0,0 +1,72 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_post_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {{skip, "Test skip"}, State}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl new file mode 100644 index 0000000000..60b1a558ae --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl @@ -0,0 +1,73 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_pre_suite_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {{skip, "Test skip"}, State}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl new file mode 100644 index 0000000000..35c990c0be --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl @@ -0,0 +1,83 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(state_update_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ State = empty_cth:init(Id, Opts),
+ [init|State].
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {Config, [pre_init_per_suite|State]}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {Config, [post_init_per_suite|State]}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State),
+ {Config, [pre_end_per_suite|State]}.
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State),
+ {Return, [post_end_per_suite|State]}.
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State),
+ {Config, [pre_init_per_group|State]}.
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State),
+ {Return, [post_init_per_group|State]}.
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State),
+ {Config, [pre_end_per_group|State]}.
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State),
+ {Return, [post_end_per_group|State]}.
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State),
+ {Config, [pre_init_per_testcase|State]}.
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State),
+ {Return, [post_end_per_testcase|State]}.
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State),
+ [on_tc_fail|State].
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State),
+ [on_tc_skip|State].
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl new file mode 100644 index 0000000000..cd561771d5 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl @@ -0,0 +1,71 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(undef_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(_Suite, _Config, _State) ->
+ lists:flaten([1,2,[3,4]]).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl new file mode 100644 index 0000000000..2ee0d7da9c --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl @@ -0,0 +1,82 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(update_config_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State),
+ {[{pre_init_per_suite,now()}|Config],State}.
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State),
+ {[{post_init_per_suite,now()}|Return],State}.
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State),
+ {[{pre_end_per_suite,now()}|Config],State}.
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State),
+ NewConfig = [{post_end_per_suite,now()}|Config],
+ {NewConfig,NewConfig}.
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State),
+ {[{pre_init_per_group,now()}|Config],State}.
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State),
+ {[{post_init_per_group,now()}|Return],State}.
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State),
+ {[{pre_end_per_group,now()}|Config],State}.
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State),
+ {[{post_end_per_group,now()}|Config],State}.
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State),
+ {[{pre_init_per_testcase,now()}|Config],State}.
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State),
+ {[{post_end_per_testcase,now()}|Config],State}.
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index e0e1f93db2..e89b6f7de6 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,11 +28,18 @@ -module(ct_master_SUITE). -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). +-define(TEMP_DIR, case os:type() of + {win32,_} -> + "c:/Temp"; + _ -> + "/tmp" + end). + %%-------------------------------------------------------------------- %% TEST SERVER CALLBACK FUNCTIONS %%-------------------------------------------------------------------- @@ -43,39 +50,84 @@ %% there will be clashes with logging processes etc). %%-------------------------------------------------------------------- init_per_suite(Config) -> - Config1 = ct_test_support:init_per_suite(Config), - Config1. + ct_test_support:init_per_suite(Config). end_per_suite(Config) -> ct_test_support:end_per_suite(Config). init_per_testcase(TestCase, Config) -> - ct_test_support:init_per_testcase(TestCase, [{master, true}|Config]). + NodeCount = 5, + NodeNames = [list_to_atom("t_"++integer_to_list(N)) || + N <- lists:seq(1, NodeCount)], + ct_test_support:init_per_testcase( + TestCase,[{node_names,NodeNames}, + {master, true}|Config]). end_per_testcase(TestCase, Config) -> + case os:type() of + {win32,_} -> + %% If this is a windows run the logs are saved to /tmp and + %% then moved to private_dir as a tar because otherwise + %% the file names become too long! :( + Files = filelib:wildcard(filename:join(?TEMP_DIR,"slave.*")), + erl_tar:create( + filename:join( + proplists:get_value(priv_dir,Config),"slaves.tar.gz"), + Files,[compressed]), + os:cmd("rm -rf "++filename:join(?TEMP_DIR,"slave.*")); + _ -> + ok + end, + ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ct_master_test]. -all(suite) -> - [ - ct_master_test - ]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %%-------------------------------------------------------------------- %% TEST CASES %%-------------------------------------------------------------------- ct_master_test(Config) when is_list(Config)-> - NodeCount = 5, + NodeNames = proplists:get_value(node_names, Config), DataDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), - NodeNames = [list_to_atom("testnode_"++integer_to_list(N)) || - N <- lists:seq(1, NodeCount)], + FileName = filename:join(PrivDir, "ct_master_spec.spec"), Suites = [master_SUITE], TSFile = make_spec(DataDir, FileName, NodeNames, Suites, Config), + ERPid = ct_test_support:start_event_receiver(Config), + spawn(ct@ancalagon, + fun() -> + dbg:tracer(),dbg:p(all,c), + dbg:tpl(erlang, spawn_link, 4,x), + receive ok -> ok end + end), + [{TSFile, ok}] = run_test(ct_master_test, FileName, Config), + + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(groups_suite_1, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + find_events(NodeNames, [{tc_start,{master_SUITE,init_per_suite}}, + {tc_start,{master_SUITE,first_testcase}}, + {tc_start,{master_SUITE,second_testcase}}, + {tc_start,{master_SUITE,third_testcase}}, + {tc_start,{master_SUITE,end_per_suite}}], + Events), + ok. %%%----------------------------------------------------------------- @@ -112,13 +164,25 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config)-> PrivDir = ?config(priv_dir, Config), LD = lists:map(fun(NodeName)-> - {logdir, NodeName, get_log_dir(PrivDir, NodeName)} + {logdir, NodeName, get_log_dir(os:type(),PrivDir, NodeName)} end, NodeNames) ++ [{logdir, master, PrivDir}], - - ct_test_support:write_testspec(N++C++S++LD++NS, FileName). - -get_log_dir(PrivDir, NodeName)-> + EvHArgs = [{cbm,ct_test_support},{trace_level,?config(trace_level,Config)}], + EH = [{event_handler,master,[?eh],EvHArgs}], + + Include = [{include,filename:join([DataDir,"master/include"])}], + + ct_test_support:write_testspec(N++Include++EH++C++S++LD++NS, FileName). + +get_log_dir({win32,_},PrivDir, NodeName)-> + case filelib:is_dir(?TEMP_DIR) of + false -> + file:make_dir(?TEMP_DIR); + _ -> + ok + end, + get_log_dir(tmp, ?TEMP_DIR,NodeName); +get_log_dir(_,PrivDir,NodeName) -> LogDir = filename:join(PrivDir, io_lib:format("slave.~p", [NodeName])), file:make_dir(LogDir), LogDir. @@ -126,11 +190,34 @@ get_log_dir(PrivDir, NodeName)-> run_test(_Name, FileName, Config)-> [{FileName, ok}] = ct_test_support:run(ct_master, run, [FileName], Config). -reformat_events(Events, EH) -> +reformat(Events, EH) -> ct_test_support:reformat(Events, EH). %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- +find_events([], _CheckEvents, _) -> + ok; +find_events([NodeName|NodeNames],CheckEvents,AllEvents) -> + find_events(NodeNames, CheckEvents, + remove_events(add_host(NodeName),CheckEvents, AllEvents, [])). + +remove_events(Node,[{Name,Data} | RestChecks], + [{?eh,#event{ name = Name, node = Node, data = Data }}|RestEvs], + Acc) -> + remove_events(Node, RestChecks, RestEvs, Acc); +remove_events(Node, Checks, [Event|RestEvs], Acc) -> + remove_events(Node, Checks, RestEvs, [Event | Acc]); +remove_events(_Node, [], [], Acc) -> + lists:reverse(Acc); +remove_events(Node, Events, [], Acc) -> + test_server:format("Could not find events: ~p in ~p for node ~p", + [Events, lists:reverse(Acc), Node]), + exit(event_not_found). + +add_host(NodeName) -> + {ok, HostName} = inet:gethostname(), + list_to_atom(atom_to_list(NodeName)++"@"++HostName). + expected_events(_)-> -[]. + []. diff --git a/lib/common_test/test/ct_master_SUITE_data/master/include/test.hrl b/lib/common_test/test/ct_master_SUITE_data/master/include/test.hrl new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/common_test/test/ct_master_SUITE_data/master/include/test.hrl diff --git a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl index e37ec3659c..032d69ad9f 100644 --- a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl @@ -28,6 +28,7 @@ -compile(export_all). -include_lib("common_test/include/ct.hrl"). +-include("test.hrl"). suite() -> []. diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl index 8c8b2d0d41..a8bd2c2189 100644 --- a/lib/common_test/test/ct_misc_1_SUITE.erl +++ b/lib/common_test/test/ct_misc_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("test_server/include/test_server_line.hrl"). -include_lib("common_test/include/ct_event.hrl"). @@ -57,13 +57,23 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [ - beam_me_up, parse_table - ]. +all() -> + [beam_me_up, {group,parse_table}]. + +groups() -> + [{parse_table,[parallel], + [parse_table_empty, parse_table_single, + parse_table_multiline_row, + parse_table_one_column_multiline, + parse_table_one_column_simple]}]. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %%-------------------------------------------------------------------- %% TEST CASES @@ -106,13 +116,6 @@ beam_me_up(Config) when is_list(Config) -> TestEvents = events_to_check(beam_me_up, 1), ok = ct_test_support:verify_events(TestEvents, Events, Config). - -parse_table(suite) -> - [parse_table_empty, parse_table_single, - parse_table_multiline_row, - parse_table_one_column_multiline, - parse_table_one_column_simple]. - parse_table_empty(Config) when is_list(Config) -> String = ["+----+-------+---------+---------+----------+------+--------+", diff --git a/lib/common_test/test/ct_repeat_1_SUITE.erl b/lib/common_test/test/ct_repeat_1_SUITE.erl index 1b4cafc9d3..e674315526 100644 --- a/lib/common_test/test/ct_repeat_1_SUITE.erl +++ b/lib/common_test/test/ct_repeat_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,24 +56,26 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - []; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [repeat_cs, - repeat_cs_and_grs, - repeat_seq, - repeat_cs_until_any_ok, - repeat_gr_until_any_ok, - repeat_cs_until_any_fail, - repeat_gr_until_any_fail, - repeat_cs_until_all_ok, - repeat_gr_until_all_ok, - repeat_cs_until_all_fail, - repeat_gr_until_all_fail, +all() -> + [repeat_cs, repeat_cs_and_grs, repeat_seq, + repeat_cs_until_any_ok, repeat_gr_until_any_ok, + repeat_cs_until_any_fail, repeat_gr_until_any_fail, + repeat_cs_until_all_ok, repeat_gr_until_all_ok, + repeat_cs_until_all_fail, repeat_gr_until_all_fail, repeat_seq_until_any_fail, - repeat_shuffled_seq_until_any_fail - ]. + repeat_shuffled_seq_until_any_fail]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl index 0cf40f106a..c7650b169c 100644 --- a/lib/common_test/test/ct_sequence_1_SUITE.erl +++ b/lib/common_test/test/ct_sequence_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,13 +56,24 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(suite) -> - [subgroup_return_fail, - subgroup_init_fail, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [subgroup_return_fail, subgroup_init_fail, subgroup_after_failed_case, case_after_subgroup_return_fail, case_after_subgroup_fail_init]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index 2e02061dec..62c5f10b7c 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,14 +56,20 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [auto_skip, user_skip]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [ - auto_skip, - user_skip - ]. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_smoke_test_SUITE.erl b/lib/common_test/test/ct_smoke_test_SUITE.erl index 05a2c20695..c3d49a5afa 100644 --- a/lib/common_test/test/ct_smoke_test_SUITE.erl +++ b/lib/common_test/test/ct_smoke_test_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -112,14 +112,22 @@ end_per_testcase(TestCase, Config) -> %% Description: Returns a description of the test suite (doc) and a %% list of all test cases in the suite (suite). %%-------------------------------------------------------------------- -all(doc) -> - ["Run smoke tests of Common Test."]; +suite() -> [{ct_hooks,[ts_install_cth]}]. -all(suite) -> - [dir1, dir2, dir1_2, - suite11, suite21, suite11_21, +all() -> + [dir1, dir2, dir1_2, suite11, suite21, suite11_21, tc111, tc211, tc111_112]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%-------------------------------------------------------------------- %% TEST CASES diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index eb85409073..9d3e6a9e59 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,13 +56,20 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - [""]; +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ts_if_1]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [ - ts_if_1 - ]. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 7bfb9ffb49..b4f1a0e71f 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -43,6 +43,14 @@ init_per_suite(Config) -> init_per_suite(Config, 50). init_per_suite(Config, Level) -> + case os:type() of + {win32, _} -> + %% Extend timeout for windows as starting node + %% can take a long time there + test_server:timetrap( 120000 * test_server:timetrap_scale_factor()); + _ -> + ok + end, case delete_old_logs(os:type(), Config) of {'EXIT',DelLogsReason} -> test_server:format(0, "Failed to delete old log directories: ~p~n", @@ -50,7 +58,13 @@ init_per_suite(Config, Level) -> _ -> ok end, + + start_slave(Config, Level). + +start_slave(Config,Level) -> [_,Host] = string:tokens(atom_to_list(node()), "@"), + + test_server:format(0, "Trying to start ~s~n", ["ct@"++Host]), case slave:start(Host, ct, []) of {error,Reason} -> test_server:fail(Reason); @@ -126,9 +140,16 @@ init_per_testcase(_TestCase, Config) -> end_per_testcase(_TestCase, Config) -> CTNode = ?config(ct_node, Config), - wait_for_ct_stop(CTNode), - ok. - + case wait_for_ct_stop(CTNode) of + %% Common test was not stopped to we restart node. + false -> + cover:stop(CTNode), + slave:stop(CTNode), + start_slave(Config,proplists:get_value(trace_level,Config)), + {fail, "Could not stop common_test"}; + true -> + ok + end. %%%----------------------------------------------------------------- %%% @@ -219,11 +240,11 @@ wait_for_ct_stop(CTNode) -> wait_for_ct_stop(0, CTNode) -> test_server:format(0, "Giving up! Stopping ~p.", [CTNode]), - ok; + false; wait_for_ct_stop(Retries, CTNode) -> case rpc:call(CTNode, erlang, whereis, [ct_util_server]) of undefined -> - ok; + true; Pid -> test_server:format(0, "Waiting for CT (~p) to finish (~p)...", [Pid,Retries]), @@ -351,13 +372,33 @@ locate({parallel,TEvs}, Node, Evs, Config) -> case Evs of [{TEH,#event{name=tc_start, node=Node, - data={M,{init_per_group,GroupName,Props}}}}, - {TEH,#event{name=tc_done, - node=Node, - data={M,{init_per_group,GroupName,Props},R}}} | Es] -> + data={M,{init_per_group, + GroupName,Props}}}}|Es] -> + %% Use dropwhile here as a tc_done from a + %% previous testcase might sneak in here + EvsG = lists:dropwhile( + fun({EH,#event{name=tc_done, + node=EvNode, + data={EvM,{init_per_group, + EvGroupName, + EvProps},EvR}}}) + when TEH == EH, EvNode == Node, EvM == M, + EvGroupName == GroupName, + EvProps == Props, + EvR == R -> + false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) + when EH == TEH, EvNode == Node -> + exit({group_init_done_not_found, + GroupName,Props}); + (_) -> + true + end, Es), + test_server:format("Found ~p!", [InitStart]), test_server:format("Found ~p!", [InitDone]), - {TEs,Es}; + {TEs,EvsG}; _ -> nomatch end; @@ -846,22 +887,49 @@ locate({TEH,tc_done,{undefined,undefined,{testcase_aborted, nomatch end; -%% matches any event of type Name -locate({TEH,Name,Data}, Node, [Ev|Evs], Config) when Data == '_' -> - case Ev of - {TEH,#event{name=Name, node=Node}} -> - {Config,Evs}; +%% Negative matching: Given two events, the first should not be present before +%% the other is matched. +locate({negative,NotMatch, Match} = Neg, Node, Evs, Config) -> + case locate(NotMatch, Node, Evs, Config) of + nomatch -> + locate(Match, Node, Evs, Config); _ -> - nomatch + exit({found_negative_event,Neg}) end; -locate({TEH,Name,Data}, Node, [Ev|Evs], Config) -> - case Ev of - {TEH,#event{name=Name, node=Node, data=Data}} -> - {Config,Evs}; - _ -> +%% matches any event of type Name +locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name, + data = EvData, + node = Node}}|Evs], + Config) -> + try match_data(Data, EvData) of + match -> + {Config,Evs} + catch _:_ -> nomatch - end. + end; + +locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) -> + nomatch. + +match_data(D,D) -> + match; +match_data('_',_) -> + match; +match_data(Fun,Data) when is_function(Fun) -> + Fun(Data); +match_data('$proplist',Proplist) -> + match_data( + fun(List) -> + lists:foreach(fun({_,_}) -> ok end,List) + end,Proplist); +match_data([H1|MatchT],[H2|ValT]) -> + match_data(H1,H2), + match_data(MatchT,ValT); +match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> + match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); +match_data([],[]) -> + match. log_events(TC, Events, PrivDir) -> LogFile = filename:join(PrivDir, atom_to_list(TC)++".events"), diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index dc399bfb4c..616c2db869 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,7 +29,7 @@ -compile(export_all). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("common_test/include/ct_event.hrl"). -define(eh, ct_test_support_eh). @@ -56,24 +56,41 @@ init_per_testcase(TestCase, Config) -> end_per_testcase(TestCase, Config) -> ct_test_support:end_per_testcase(TestCase, Config). -all(doc) -> - ["Run smoke tests of Common Test."]; - -all(suite) -> - [all_suites, skip_all_suites, - suite, skip_suite, - all_testcases, skip_all_testcases, - testcase, skip_testcase, - all_groups, skip_all_groups, - group, skip_group, - group_all_testcases, skip_group_all_testcases, - group_testcase, skip_group_testcase, - topgroup, - subgroup, skip_subgroup, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [all_suites, skip_all_suites, suite, skip_suite, + all_testcases, skip_all_testcases, testcase, + skip_testcase, all_groups, skip_all_groups, group, + skip_group, group_all_testcases, + skip_group_all_testcases, group_testcase, + skip_group_testcase, topgroup, subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, - sub_skipped_by_top, - testcase_in_multiple_groups]. + sub_skipped_by_top, testcase_in_multiple_groups, + order_of_tests_in_multiple_dirs_no_merge_tests, + order_of_tests_in_multiple_suites_no_merge_tests, + order_of_suites_in_multiple_dirs_no_merge_tests, + order_of_groups_in_multiple_dirs_no_merge_tests, + order_of_groups_in_multiple_suites_no_merge_tests, + order_of_tests_in_multiple_dirs, + order_of_tests_in_multiple_suites, + order_of_suites_in_multiple_dirs, + order_of_groups_in_multiple_dirs, + order_of_groups_in_multiple_suites, + order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + order_of_tests_in_multiple_suites_with_skip, + all_plus_one_tc_no_merge_tests, + all_plus_one_tc]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %%-------------------------------------------------------------------- %% TEST CASES @@ -366,6 +383,223 @@ testcase_in_multiple_groups(Config) when is_list(Config) -> setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). %%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir2,groups_22_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {suites,TestDir1,groups_12_SUITE}, + {suites,TestDir2,groups_22_SUITE}, + {suites,TestDir1,groups_11_SUITE}], + + setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{merge_tests, false}, + {groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir2,groups_22_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_suites_no_merge_tests(Config) + when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir1,groups_11_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config) + when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests, false}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}, + {cases,TestDir1,groups_11_SUITE,[testcase_2]}, + {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}], + + setup_and_execute( + order_of_tests_in_multiple_suites_with_skip_no_merge_tests, + TestSpec, Config). + + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir2,groups_22_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_dirs, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}], + + setup_and_execute(order_of_tests_in_multiple_suites, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_suites_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{suites,TestDir1,groups_12_SUITE}, + {suites,TestDir2,groups_22_SUITE}, + {suites,TestDir1,groups_11_SUITE}], + + setup_and_execute(order_of_suites_in_multiple_dirs, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_dirs(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestDir2 = filename:join(DataDir, "groups_2"), + TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir2,groups_22_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_dirs, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_groups_in_multiple_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a}, + {groups,TestDir1,groups_11_SUITE,test_group_1a}, + {groups,TestDir1,groups_12_SUITE,test_group_1b}], + + setup_and_execute(order_of_groups_in_multiple_suites, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]}, + {cases,TestDir1,groups_11_SUITE,[testcase_1]}, + {cases,TestDir1,groups_12_SUITE,[testcase_1b]}, + {cases,TestDir1,groups_11_SUITE,[testcase_2]}, + {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}], + + setup_and_execute(order_of_tests_in_multiple_suites_with_skip, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_plus_one_tc_no_merge_tests(Config) when is_list(Config) -> + + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{merge_tests,false}, + {suites,TestDir1,groups_12_SUITE}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}], + + setup_and_execute(all_plus_one_tc_no_merge_tests, + TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_plus_one_tc(Config) when is_list(Config) -> + + DataDir = ?config(data_dir, Config), + + TestDir1 = filename:join(DataDir, "groups_1"), + TestSpec = [{suites,TestDir1,groups_12_SUITE}, + {cases,TestDir1,groups_12_SUITE,[testcase_1a]}], + + setup_and_execute(all_plus_one_tc, + TestSpec, Config). + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -428,6 +662,719 @@ events_to_check(_, 0) -> events_to_check(Test, N) -> test_events(Test) ++ events_to_check(Test, N-1). +test_events(all_suites) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{simple_2_SUITE,init_per_suite}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_suites) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {?eh,tc_user_skip,{simple_2_SUITE,all,"SKIPPED!"}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(suite) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_suite) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,test_stats,{1,0,{0,0}}}, + {negative,{?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{simple_1_SUITE,init_per_suite}}, + {?eh,tc_user_skip,{simple_1_SUITE,tc1,"SKIPPED!"}}, + {?eh,tc_start,{simple_1_SUITE,tc2}}, + {?eh,tc_start,{simple_1_SUITE,end_per_suite}}, + + {?eh,tc_start,{simple_2_SUITE,init_per_suite}}, + {?eh,tc_user_skip,{simple_2_SUITE,tc2,"SKIPPED!"}}, + {?eh,tc_start,{simple_2_SUITE,tc1}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_start,{simple_2_SUITE,end_per_suite}}, + + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(all_groups) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,test_stats,{12,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_all_groups) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{2,0,{1,0}}}, + {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(group_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,test_stats,{1,0,{0,0}}}, + {negative,{?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}}, + + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_group_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{1,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, + + {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1b,[]}}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1b,[]},'_'}}, + + {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(topgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{12,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {?eh,tc_user_skip, + {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,test_stats,{10,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup_all_testcases) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}} + ], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(subgroup_testcase) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}} + ], + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_3,[]}}} + ], + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]}, + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(skip_subgroup_testcase) -> + [ + + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, + {parallel, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, + {?eh,tc_user_skip, {groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{1,0}}}, + {?eh,tc_user_skip, {groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,test_stats,{1,0,{2,0}}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], + {shuffle, + [{?eh,tc_start, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]}}}, + {?eh,tc_done, + {groups_12_SUITE,{init_per_group,test_group_8, + [{shuffle,'_'},sequence]},ok}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8, + [shuffle,sequence]},ok}} + ]}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} + ]}, + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done, + {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + + ]; + +test_events(sub_skipped_by_top) -> + [ + {?eh,start_logging,'_'}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, + + {negative, + {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}}, + + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; + +test_events(testcase_in_multiple_groups) -> + []; + +test_events(order_of_tests_in_multiple_dirs_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_22_SUITE,testcase_1}}, + {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done, {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_tests_in_multiple_suites_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_suites_in_multiple_dirs_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_dirs_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_suites_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,stop_logging,[]}]; +test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {?eh,stop_logging,[]} + ]; + +test_events(order_of_tests_in_multiple_dirs) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done, + {groups_12_SUITE,testcase_1a, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done, + {groups_12_SUITE,testcase_1b, + {failed,{error,{test_case_failed,no_group_data}}}}}, + {?eh,tc_start,{groups_22_SUITE,testcase_1}}, + {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_tests_in_multiple_suites) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + + {?eh,tc_start,{groups_12_SUITE,testcase_1b}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}}, + + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,stop_logging,[]} + ]; +test_events(order_of_suites_in_multiple_dirs) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + + {?eh,tc_start,{groups_22_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}}, + {?eh,tc_start,{groups_22_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_dirs) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,stop_logging,[]}]; +test_events(order_of_groups_in_multiple_suites) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}}, + {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}}, + + {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}}, + {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}}, + + {?eh,stop_logging,[]}]; + +test_events(order_of_tests_in_multiple_suites_with_skip) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,testcase_1a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1}}, + {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_2}}, + {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}}, + {?eh,stop_logging,[]} + ]; + +test_events(all_plus_one_tc_no_merge_tests) -> + + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,stop_logging,[]} + ]; + +test_events(all_plus_one_tc) -> + + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {negative,{?eh,tc_start,{groups_12_SUITE,init_per_suite}}, + {?eh,stop_logging,[]}} + ]; + test_events(_) -> [ ]. diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index 413ef21df3..8a4853e070 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1,3 +1,3 @@ -COMMON_TEST_VSN = 1.5.1 +COMMON_TEST_VSN = 1.5.3 |