aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2017-03-06 08:51:22 +0100
committerSiri Hansen <[email protected]>2017-03-06 08:51:22 +0100
commit0e0155410734e04556ceecb1222dd2ef956b4177 (patch)
treea714e355d78f4c96f1644ab64783a1a0a32d4da7
parent0438c6c2c5c4a447ab3a47a443968488a0d835eb (diff)
parent50cf93672b8abb21f1455d41d333481071e6194e (diff)
downloadotp-0e0155410734e04556ceecb1222dd2ef956b4177.tar.gz
otp-0e0155410734e04556ceecb1222dd2ef956b4177.tar.bz2
otp-0e0155410734e04556ceecb1222dd2ef956b4177.zip
Merge branch 'siri/ct_hooks/callbacks-on-skip/OTP-10599' into maint
* siri/ct_hooks/callbacks-on-skip/OTP-10599: [ct] Update built-in ct hooks with new Suite parameter [cth_surefire] Handle skips from test spec [ct] Remove excessive skipped/failed tag in hook function [ct] Send tc_start event on force_stop and failed sequence [ct] Correctly handle process died in init and end_per_testcase [ct] Fix faulty hook callbacks for skipped tests Add dummy end_per_suite/1 [ct] Fix hooks and fail when one of init/end_per_* does not exit [ct] Add Suite argument to hook callback functions [ct] Fix function_clause in ct_framework when hook function crashes
-rw-r--r--erts/test/upgrade_SUITE.erl4
-rw-r--r--lib/common_test/doc/src/common_test_app.xml26
-rw-r--r--lib/common_test/doc/src/ct_hooks.xml90
-rw-r--r--lib/common_test/doc/src/ct_hooks_chapter.xml74
-rw-r--r--lib/common_test/doc/src/write_test_chapter.xml2
-rw-r--r--lib/common_test/src/ct_framework.erl161
-rw-r--r--lib/common_test/src/ct_groups.erl14
-rw-r--r--lib/common_test/src/ct_hooks.erl141
-rw-r--r--lib/common_test/src/cth_conn_log.erl8
-rw-r--r--lib/common_test/src/cth_log_redirect.erl28
-rw-r--r--lib/common_test/src/cth_surefire.erl54
-rw-r--r--lib/common_test/src/test_server.erl10
-rw-r--r--lib/common_test/src/test_server_ctrl.erl116
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl8
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl1097
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl62
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl43
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl51
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl54
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl39
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl122
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl32
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl81
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl6
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl42
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl45
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec8
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl106
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl182
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl53
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl64
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl53
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl79
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl53
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl40
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl40
-rw-r--r--lib/common_test/test/ct_repeat_testrun_SUITE.erl5
-rw-r--r--lib/common_test/test/ct_surefire_SUITE.erl129
-rw-r--r--lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec2
-rw-r--r--lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec2
-rw-r--r--lib/common_test/test/ct_test_server_if_1_SUITE.erl1
-rw-r--r--lib/common_test/test/ct_test_support.erl24
-rw-r--r--lib/dialyzer/test/abstract_SUITE.erl6
-rw-r--r--lib/ssl/test/ssl_npn_hello_SUITE.erl4
54 files changed, 2918 insertions, 743 deletions
diff --git a/erts/test/upgrade_SUITE.erl b/erts/test/upgrade_SUITE.erl
index 174c028ac7..f93da8955f 100644
--- a/erts/test/upgrade_SUITE.erl
+++ b/erts/test/upgrade_SUITE.erl
@@ -54,6 +54,10 @@ init_per_suite(Config) ->
Config
end.
+end_per_suite(_Config) ->
+ %% This function is required since init_per_suite/1 exists.
+ ok.
+
init_per_testcase(Case,Config) ->
PrivDir = filename:join([proplists:get_value(data_dir,Config),priv_dir,Case]),
CreateDir = filename:join([PrivDir,create]),
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml
index 48ffe653e4..d407a0a53f 100644
--- a/lib/common_test/doc/src/common_test_app.xml
+++ b/lib/common_test/doc/src/common_test_app.xml
@@ -224,7 +224,9 @@
</type>
<desc>
- <p>OPTIONAL</p>
+ <p>OPTIONAL; if this function is defined, then <seealso
+ marker="#Module:end_per_suite-1"><c>end_per_suite/1</c></seealso>
+ must also be defined.</p>
<p>This configuration function is called as the first function in the
suite. It typically contains initializations that are common for
@@ -256,7 +258,9 @@
</type>
<desc>
- <p>OPTIONAL</p>
+ <p>OPTIONAL; if this function is defined, then <seealso
+ marker="#Module:init_per_suite-1"><c>init_per_suite/1</c></seealso>
+ must also be defined.</p>
<p>This function is called as the last test case in the
suite. It is meant to be used for cleaning up after
@@ -360,7 +364,9 @@
</type>
<desc>
- <p>OPTIONAL</p>
+ <p>OPTIONAL; if this function is defined, then <seealso
+ marker="#Module:end_per_group-2"><c>end_per_group/2</c></seealso>
+ must also be defined.</p>
<p>This configuration function is called before execution of a
test case group. It typically contains initializations that are
@@ -396,7 +402,9 @@
</type>
<desc>
- <p>OPTIONAL</p>
+ <p>OPTIONAL; if this function is defined, then <seealso
+ marker="#Module:init_per_group-2"><c>init_per_group/2</c></seealso>
+ must also be defined.</p>
<p>This function is called after the execution of a test case group
is finished. It is meant to be used for cleaning up after
@@ -427,7 +435,10 @@
</type>
<desc>
- <p>OPTIONAL</p>
+ <p>OPTIONAL; if this function is defined,
+ then <seealso marker="#Module:end_per_testcase-2">
+ <c>end_per_testcase/2</c></seealso> must also be
+ defined.</p>
<p>This function is called before each test case. Argument
<c>TestCase</c> is the test case name, and
@@ -454,7 +465,10 @@
</type>
<desc>
- <p>OPTIONAL</p>
+ <p>OPTIONAL; if this function is defined,
+ then <seealso marker="#Module:init_per_testcase-2">
+ <c>init_per_testcase/2</c></seealso> must also be
+ defined.</p>
<p>This function is called after each test case, and can be used
to clean up after
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index c2cf29c530..a085f30262 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -208,9 +208,10 @@
</func>
<func>
- <name>Module:pre_init_per_group(GroupName, InitData, CTHState) -&gt; Result</name>
+ <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -&gt; Result</name>
<fsummary>Called before init_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>InitData = Config | SkipOrFail</v>
<v>Config = NewConfig = [{Key,Value}]</v>
@@ -231,13 +232,19 @@
but for function
<seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>
instead.</p>
+
+ <p>If <c>Module:pre_init_per_group/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_init_per_group(GroupName,
+ InitData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after init_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -258,13 +265,19 @@
but for function
<seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>
instead.</p>
+
+ <p>If <c>Module:post_init_per_group/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_init_per_group(GroupName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -&gt; Result</name>
+ <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -&gt; Result</name>
<fsummary>Called before init_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
<v>InitData = Config | SkipOrFail</v>
<v>Config = NewConfig = [{Key,Value}]</v>
@@ -286,6 +299,11 @@
<seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>
instead.</p>
+ <p>If <c>Module:pre_init_per_testcase/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_init_per_testcase(TestcaseName,
+ InitData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
+
<p>CTHs cannot be added here right now. That feature may be added in
a later release, but it would right now break backwards
compatibility.</p>
@@ -293,9 +311,10 @@
</func>
<func>
- <name>Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after init_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -316,15 +335,21 @@
but for function
<seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>
instead.</p>
+
+ <p>If <c>Module:post_init_per_testcase/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_init_per_testcase(TestcaseName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -&gt; Result</name>
+ <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -&gt; Result</name>
<fsummary>Called before end_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
- <v>InitData = Config</v>
+ <v>EndData = Config</v>
<v>Config = NewConfig = [{Key,Value}]</v>
<v>CTHState = NewCTHState = term()</v>
<v>Result = {NewConfig, NewCTHState}</v>
@@ -345,14 +370,20 @@
<p>This function can not change the result of the test case by returning skip or fail
tuples, but it may insert items in <c>Config</c> that can be read in
- <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/4</c>.</p>
+ <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p>
+
+ <p>If <c>Module:pre_end_per_testcase/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_end_per_testcase(TestcaseName,
+ EndData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after end_per_testcase.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestcaseName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -373,13 +404,19 @@
but for function
<seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso>
instead.</p>
+
+ <p>If <c>Module:post_end_per_testcase/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_end_per_testcase(TestcaseName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:pre_end_per_group(GroupName, EndData, CTHState) -&gt; Result</name>
+ <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -&gt; Result</name>
<fsummary>Called before end_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>EndData = Config | SkipOrFail</v>
<v>Config = NewConfig = [{Key,Value}]</v>
@@ -400,13 +437,19 @@
but for function
<seealso marker="common_test#Module:end_per_group-2"><c>end_per_group</c></seealso>
instead.</p>
+
+ <p>If <c>Module:pre_end_per_group/4</c> is not exported, common_test
+ will attempt to call <c>Module:pre_end_per_group(GroupName,
+ EndData, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -&gt; Result</name>
+ <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -&gt; Result</name>
<fsummary>Called after end_per_group.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>GroupName = atom()</v>
<v>Config = [{Key,Value}]</v>
<v>Return = NewReturn = Config | SkipOrFail | term()</v>
@@ -427,6 +470,11 @@
but for function
<seealso marker="common_test#Module:end_per_group-2">end_per_group</seealso>
instead.</p>
+
+ <p>If <c>Module:post_end_per_group/5</c> is not exported, common_test
+ will attempt to call <c>Module:post_end_per_group(GroupName,
+ Config, Return, CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
@@ -485,9 +533,10 @@
</func>
<func>
- <name>Module:on_tc_fail(TestName, Reason, CTHState) -&gt; NewCTHState</name>
+ <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -&gt; NewCTHState</name>
<fsummary>Called after the CTH scope ends.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>
<v>FuncName = atom()</v>
<v>GroupName = atom()</v>
@@ -505,7 +554,7 @@
<item><p>If <c>init_per_suite</c> fails, this function is called after
<seealso marker="#Module:post_init_per_suite-4"><c>post_init_per_suite</c></seealso>.</p></item>
<item><p>If a test case fails, this funcion is called after
- <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item>
+ <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>
</list>
<p>If the failed test case belongs to a test case group, the first
@@ -519,13 +568,19 @@
For details, see section
<seealso marker="event_handler_chapter#events">Event Handling</seealso>
in the User's Guide.</p>
+
+ <p>If <c>Module:on_tc_fail/4</c> is not exported, common_test
+ will attempt to call <c>Module:on_tc_fail(TestName, Reason,
+ CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
<func>
- <name>Module:on_tc_skip(TestName, Reason, CTHState) -&gt; NewCTHState</name>
+ <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -&gt; NewCTHState</name>
<fsummary>Called after the CTH scope ends.</fsummary>
<type>
+ <v>SuiteName = atom()</v>
<v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>
<v>FuncName = atom()</v>
<v>GroupName = atom()</v>
@@ -542,9 +597,9 @@
<list type="bulleted">
<item><p>If <c>init_per_group</c> is skipped, this function is
called after
- <seealso marker="#Module:post_init_per_group-4"><c>post_init_per_group</c></seealso>.</p></item>
+ <seealso marker="#Module:post_init_per_group-5"><c>post_init_per_group</c></seealso>.</p></item>
<item><p>If a test case is skipped, this function is called after
- <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item>
+ <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>
</list>
<p>If the skipped test case belongs to a test case group, the first
@@ -559,6 +614,11 @@
For details, see section
<seealso marker="event_handler_chapter#events">Event Handling</seealso>
in the User's Guide.</p>
+
+ <p>If <c>Module:on_tc_skip/4</c> is not exported, common_test
+ will attempt to call <c>Module:on_tc_skip(TestName, Reason,
+ CTHState)</c> instead. This is for backwards
+ compatibility.</p>
</desc>
</func>
diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml
index 0e4c35e11f..bfad96e489 100644
--- a/lib/common_test/doc/src/ct_hooks_chapter.xml
+++ b/lib/common_test/doc/src/ct_hooks_chapter.xml
@@ -38,7 +38,7 @@
extensions of the default behavior of <c>Common Test</c> using hooks
before and after all test suite calls. CTHs allow advanced <c>Common Test</c>
users to abstract out behavior that is common to multiple test suites
- without littering all test suites with library calls. this can be used
+ without littering all test suites with library calls. This can be used
for logging, starting, and monitoring external systems,
building C files needed by the tests, and so on.</p>
@@ -175,10 +175,10 @@
<row>
<cell><seealso marker="common_test#Module:init_per_group-2">
init_per_group/2</seealso></cell>
- <cell><seealso marker="ct_hooks#Module:post_init_per_group-4">
- post_init_per_group/4</seealso> is called</cell>
- <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4">
- post_end_per_group/4</seealso> has been called for that group</cell>
+ <cell><seealso marker="ct_hooks#Module:post_init_per_group-5">
+ post_init_per_group/5</seealso> is called</cell>
+ <cell><seealso marker="ct_hooks#Module:post_end_per_group-5">
+ post_end_per_group/5</seealso> has been called for that group</cell>
</row>
<tcaption>Scope of a CTH</tcaption>
</table>
@@ -245,16 +245,18 @@
</list>
<p>
- This is done in the CTH functions called pre_&lt;name of function&gt;.
- These functions take the same three arguments, <c>Name</c>,
+ This is done in the CTH functions called <c>pre_&lt;name of function&gt;</c>.
+ These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),
<c>Config</c>, and <c>CTHState</c>. The return value of the CTH function
is always a combination of a result for the suite/group/test and an
updated <c>CTHState</c>.</p>
<p>To let the test suite continue on executing, return the configuration
- list that you want the test to use as the result. To skip or
- fail the test, return a tuple with <c>skip</c> or <c>fail</c>, and a reason
- as the result.</p>
+ list that you want the test to use as the result.</p>
+
+ <p>All pre hooks, except <c>pre_end_per_testcase/4</c>, can
+ skip or fail the test by returning a tuple with <c>skip</c> or
+ <c>fail</c>, and a reason as the result.</p>
<p><em>Example:</em></p>
<code>
@@ -290,7 +292,7 @@
<p>
This is done in the CTH functions called <c>post_&lt;name of function&gt;</c>.
- These functions take the same four arguments, <c>Name</c>,
+ These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),
<c>Config</c>, <c>Return</c>, and <c>CTHState</c>. <c>Config</c> in this
case is the same <c>Config</c> as the testcase is called with.
<c>Return</c> is the value returned by the testcase. If the testcase
@@ -308,7 +310,7 @@
<p><em>Example:</em></p>
<code>
- post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -&gt;
+ post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) -&gt;
case db:check_consistency() of
true ->
%% DB is good, pass the test.
@@ -317,7 +319,7 @@
%% DB is not good, mark as skipped instead of failing
{{skip, "DB is inconsisten!"}, CTHState}
end;
- post_end_per_testcase(_TC, Config, Return, CTHState) -&gt;
+ post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) -&gt;
%% Do nothing if tc does not crash.
{Return, CTHState}.</code>
@@ -331,8 +333,8 @@
<title>Skip and Fail Hooks</title>
<p>
After any post hook has been executed for all installed CTHs,
- <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso>
- or <seealso marker="ct_hooks#Module:on_tc_skip-3">on_tc_skip</seealso>
+ <seealso marker="ct_hooks#Module:on_tc_fail-4">on_tc_fail</seealso>
+ or <seealso marker="ct_hooks#Module:on_tc_skip-4">on_tc_skip</seealso>
is called if the testcase failed or was skipped, respectively.
You cannot affect the outcome of the tests any further at this point.
</p>
@@ -389,18 +391,18 @@
-export([pre_end_per_suite/3]).
-export([post_end_per_suite/4]).
- -export([pre_init_per_group/3]).
- -export([post_init_per_group/4]).
- -export([pre_end_per_group/3]).
- -export([post_end_per_group/4]).
+ -export([pre_init_per_group/4]).
+ -export([post_init_per_group/5]).
+ -export([pre_end_per_group/4]).
+ -export([post_end_per_group/5]).
- -export([pre_init_per_testcase/3]).
- -export([post_init_per_testcase/4]).
- -export([pre_end_per_testcase/3]).
- -export([post_end_per_testcase/4]).
+ -export([pre_init_per_testcase/4]).
+ -export([post_init_per_testcase/5]).
+ -export([pre_end_per_testcase/4]).
+ -export([post_end_per_testcase/5]).
- -export([on_tc_fail/3]).
- -export([on_tc_skip/3]).
+ -export([on_tc_fail/4]).
+ -export([on_tc_skip/4]).
-export([terminate/1]).
@@ -435,46 +437,46 @@
total = State#state.total + State#state.suite_total } }.
%% @doc Called before each init_per_group.
- pre_init_per_group(Group,Config,State) ->
+ pre_init_per_group(Suite,Group,Config,State) ->
{Config, State}.
%% @doc Called after each init_per_group.
- post_init_per_group(Group,Config,Return,State) ->
+ post_init_per_group(Suite,Group,Config,Return,State) ->
{Return, State}.
%% @doc Called before each end_per_group.
- pre_end_per_group(Group,Config,State) ->
+ pre_end_per_group(Suite,Group,Config,State) ->
{Config, State}.
%% @doc Called after each end_per_group.
- post_end_per_group(Group,Config,Return,State) ->
+ post_end_per_group(Suite,Group,Config,Return,State) ->
{Return, State}.
%% @doc Called before each init_per_testcase.
- pre_init_per_testcase(TC,Config,State) ->
+ pre_init_per_testcase(Suite,TC,Config,State) ->
{Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.
%% Called after each init_per_testcase (immediately before the test case).
- post_init_per_testcase(TC,Config,Return,State) ->
+ post_init_per_testcase(Suite,TC,Config,Return,State) ->
{Return, State}
%% @doc Called before each end_per_testcase (immediately after the test case).
- pre_end_per_testcase(TC,Config,State) ->
+ pre_end_per_testcase(Suite,TC,Config,State) ->
{Config, State}.
%% @doc Called after each end_per_testcase.
- post_end_per_testcase(TC,Config,Return,State) ->
- TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)},
+ post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)},
{Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.
%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
%% post_end_per_group and post_end_per_testcase if the suite, group or test case failed.
- on_tc_fail(TC, Reason, State) ->
+ on_tc_fail(Suite, TC, Reason, State) ->
State.
%% @doc Called when a test case is skipped by either user action
%% or due to an init function failing.
- on_tc_skip(TC, Reason, State) ->
+ on_tc_skip(Suite, TC, Reason, State) ->
State.
%% @doc Called when the scope of the CTH is done
diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml
index f70bdb16c5..6a0d87bcaf 100644
--- a/lib/common_test/doc/src/write_test_chapter.xml
+++ b/lib/common_test/doc/src/write_test_chapter.xml
@@ -566,7 +566,7 @@
for the test cases in the group. After execution of the group is finished, function
<seealso marker="common_test#Module:end_per_group-2"><c>end_per_group(GroupName, Config)</c></seealso>
is called. This function is meant to be used for cleaning up after
- <c>init_per_group/2</c>.</p>
+ <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p>
<p>Whenever a group is executed, if <c>init_per_group</c> and
<c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 291a4d716c..43f1c9de0f 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -52,6 +52,10 @@
%%%
%%% @doc Test server framework callback, called by the test_server
%%% when a new test case is started.
+init_tc(_,{end_per_testcase_not_run,_},[Config]) ->
+ %% Testcase is completed (skipped or failed), but end_per_testcase
+ %% is not run - don't call pre-hook.
+ {ok,[Config]};
init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Config),
@@ -62,7 +66,7 @@ init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->
Other
end;
-init_tc(Mod,Func0,Args) ->
+init_tc(Mod,Func0,Args) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
{Func,HookFunc} = case Func0 of
@@ -84,12 +88,15 @@ init_tc(Mod,Func0,Args) ->
andalso Func=/=end_per_group
andalso ct_util:get_testdata(skip_rest) of
true ->
+ initialize(false,Mod,Func,Args),
{auto_skip,"Repeated test stopped by force_stop option"};
_ ->
case ct_util:get_testdata(curr_tc) of
{Suite,{suite0_failed,{require,Reason}}} ->
+ initialize(false,Mod,Func,Args),
{auto_skip,{require_failed_in_suite0,Reason}};
{Suite,{suite0_failed,_}=Failure} ->
+ initialize(false,Mod,Func,Args),
{fail,Failure};
_ ->
ct_util:update_testdata(curr_tc,
@@ -118,16 +125,14 @@ init_tc(Mod,Func0,Args) ->
end,
init_tc1(Mod,Suite,Func,HookFunc,Args);
{failed,Seq,BadFunc} ->
- {auto_skip,{sequence_failed,Seq,BadFunc}}
+ initialize(false,Mod,Func,Args),
+ {auto_skip,{sequence_failed,Seq,BadFunc}}
end
end
end.
init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={?MODULE,error_in_suite}}),
+ initialize(false,?MODULE,error_in_suite),
_ = ct_suite_init(?MODULE,error_in_suite,[],Config0),
case ?val(error,Config0) of
undefined ->
@@ -177,27 +182,21 @@ init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) ->
ct_config:delete_default_config(testcase),
HookFunc
end,
- Initialize = fun() ->
- ct_logs:init_tc(false),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}})
- end,
case add_defaults(Mod,Func,AllGroups) of
Error = {suite0_failed,_} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
ct_util:set_testdata({curr_tc,{Suite,Error}}),
{error,Error};
Error = {group0_failed,_} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
{auto_skip,Error};
Error = {testcase0_failed,_} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
{auto_skip,Error};
{SuiteInfo,MergeResult} ->
case MergeResult of
{error,Reason} ->
- Initialize(),
+ initialize(false,Mod,FuncSpec),
{fail,Reason};
_ ->
init_tc2(Mod,Suite,Func,HookFunc1,
@@ -236,11 +235,8 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
Conns ->
ct_util:silence_connections(Conns)
end,
- ct_logs:init_tc(Func == init_per_suite),
FuncSpec = group_or_func(Func,Config),
- ct_event:notify(#event{name=tc_start,
- node=node(),
- data={Mod,FuncSpec}}),
+ initialize((Func==init_per_suite),Mod,FuncSpec),
case catch configure(MergedInfo,MergedInfo,SuiteInfo,
FuncSpec,[],Config) of
@@ -268,6 +264,18 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->
end
end.
+initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) ->
+ initialize(RefreshLogs,Mod,group_or_func(Func,Config));
+initialize(RefreshLogs,Mod,Func,_) ->
+ initialize(RefreshLogs,Mod,Func).
+
+initialize(RefreshLogs,Mod,FuncSpec) ->
+ ct_logs:init_tc(RefreshLogs),
+ ct_event:notify(#event{name=tc_start,
+ node=node(),
+ data={Mod,FuncSpec}}).
+
+
ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) ->
case ct_hooks:init_tc(Suite,HookFunc,Config) of
NewConfig when is_list(NewConfig) ->
@@ -675,22 +683,35 @@ end_tc(Mod,Func,{Result,[Args]}, Return) ->
end_tc(Mod,Func,self(),Result,Args,Return).
end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) ->
- %% in case Mod == ct_framework, lookup the suite name
- Suite = get_suite_name(Mod, Args),
- case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
- '$ct_no_change' ->
- ok;
- HookResult ->
- HookResult
+ case end_hook_func(IPTC,Return,IPTC) of
+ undefined -> ok;
+ _ ->
+ %% in case Mod == ct_framework, lookup the suite name
+ Suite = get_suite_name(Mod, Args),
+ case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of
+ '$ct_no_change' ->
+ ok;
+ HookResult ->
+ HookResult
+ end
end;
end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
%% in case Mod == ct_framework, lookup the suite name
Suite = get_suite_name(Mod, Args),
- {EPTC,Func} = case Func0 of
- {end_per_testcase,F} -> {true,F};
- _ -> {false,Func0}
- end,
+ {Func,FuncSpec,HookFunc} =
+ case Func0 of
+ {end_per_testcase_not_run,F} ->
+ %% Testcase is completed (skipped or failed), but
+ %% end_per_testcase is not run - don't call post-hook.
+ {F,F,undefined};
+ {end_per_testcase,F} ->
+ {F,F,Func0};
+ _ ->
+ FS = group_or_func(Func0,Args),
+ HF = end_hook_func(Func0,Return,FS),
+ {Func0,FS,HF}
+ end,
test_server:timetrap_cancel(),
@@ -717,20 +738,18 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
end,
ct_util:delete_suite_data(last_saved_config),
- {FuncSpec,HookFunc} =
- if not EPTC ->
- FS = group_or_func(Func,Args),
- {FS,FS};
- true ->
- {Func,Func0}
- end,
{Result1,FinalNotify} =
- case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
- '$ct_no_change' ->
- {ok,Result};
- HookResult ->
- {HookResult,HookResult}
- end,
+ case HookFunc of
+ undefined ->
+ {ok,Result};
+ _ ->
+ case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of
+ '$ct_no_change' ->
+ {ok,Result};
+ HookResult ->
+ {HookResult,HookResult}
+ end
+ end,
FinalResult =
case get('$test_server_framework_test') of
undefined ->
@@ -821,6 +840,34 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->
end,
FinalResult.
+%% This is to make sure that no post_init_per_* is ever called if the
+%% corresponding pre_init_per_* was not called.
+%% The skip or fail reasons are those that can be returned from
+%% init_tc above in situations where we never came to call
+%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call
+%% ct_hooks:init_tc for init_per_suite, and thus we must not call
+%% ct_hooks:end_tc for init_per_suite either.
+end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) ->
+ undefined;
+end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) ->
+ undefined;
+end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) ->
+ undefined;
+end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_)
+ when InfoFuncError==testcase0_failed;
+ InfoFuncError==require_failed ->
+ undefined;
+end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_)
+ when InfoFuncError==group0_failed;
+ InfoFuncError==require_failed ->
+ undefined;
+end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) ->
+ undefined;
+end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) ->
+ undefined;
+end_hook_func(_,_,Default) ->
+ Default.
+
%% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} |
%% {testcase_aborted,Reason} | testcase_aborted_or_killed |
%% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} |
@@ -1339,25 +1386,25 @@ report(What,Data) ->
ok;
tc_done ->
{Suite,{Func,GrName},Result} = Data,
- Data1 = if GrName == undefined -> {Suite,Func,Result};
- true -> Data
- end,
+ FuncSpec = if GrName == undefined -> Func;
+ true -> {Func,GrName}
+ end,
%% Register the group leader for the process calling the report
%% function, making it possible for a hook function to print
%% in the test case log file
ReportingPid = self(),
ct_logs:register_groupleader(ReportingPid, group_leader()),
case Result of
- {failed, _} ->
- ct_hooks:on_tc_fail(What, Data1);
- {skipped,{failed,{_,init_per_testcase,_}}} ->
- ct_hooks:on_tc_skip(tc_auto_skip, Data1);
- {skipped,{require_failed,_}} ->
- ct_hooks:on_tc_skip(tc_auto_skip, Data1);
- {skipped,_} ->
- ct_hooks:on_tc_skip(tc_user_skip, Data1);
- {auto_skipped,_} ->
- ct_hooks:on_tc_skip(tc_auto_skip, Data1);
+ {failed, Reason} ->
+ ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason});
+ {skipped,{failed,{_,init_per_testcase,_}}=Reason} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
+ {skipped,{require_failed,_}=Reason} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
+ {skipped,Reason} ->
+ ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason});
+ {auto_skipped,Reason} ->
+ ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});
_Else ->
ok
end,
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
index 1375e7dcc7..1c9faf6a70 100644
--- a/lib/common_test/src/ct_groups.erl
+++ b/lib/common_test/src/ct_groups.erl
@@ -442,17 +442,21 @@ make_conf(Mod, Name, Props, TestSpec) ->
ok
end,
{InitConf,EndConf,ExtraProps} =
- case erlang:function_exported(Mod,init_per_group,2) of
- true ->
- {{Mod,init_per_group},{Mod,end_per_group},[]};
- false ->
+ case {erlang:function_exported(Mod,init_per_group,2),
+ erlang:function_exported(Mod,end_per_group,2)} of
+ {false,false} ->
ct_logs:log("TEST INFO", "init_per_group/2 and "
"end_per_group/2 missing for group "
"~w in ~w, using default.",
[Name,Mod]),
{{ct_framework,init_per_group},
{ct_framework,end_per_group},
- [{suite,Mod}]}
+ [{suite,Mod}]};
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ {{Mod,init_per_group},{Mod,end_per_group},[]}
end,
{conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}.
diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl
index c9a4abb5ee..60d1ea2b1c 100644
--- a/lib/common_test/src/ct_hooks.erl
+++ b/lib/common_test/src/ct_hooks.erl
@@ -92,15 +92,17 @@ init_tc(Mod, end_per_suite, Config) ->
call(fun call_generic/3, Config, [pre_end_per_suite, Mod]);
init_tc(Mod, {init_per_group, GroupName, Properties}, Config) ->
maybe_start_locker(Mod, GroupName, Properties),
- call(fun call_generic/3, Config, [pre_init_per_group, GroupName]);
-init_tc(_Mod, {end_per_group, GroupName, _}, Config) ->
- call(fun call_generic/3, Config, [pre_end_per_group, GroupName]);
-init_tc(_Mod, {init_per_testcase,TC}, Config) ->
- call(fun call_generic/3, Config, [pre_init_per_testcase, TC]);
-init_tc(_Mod, {end_per_testcase,TC}, Config) ->
- call(fun call_generic/3, Config, [pre_end_per_testcase, TC]);
-init_tc(_Mod, TC = error_in_suite, Config) ->
- call(fun call_generic/3, Config, [pre_init_per_testcase, TC]).
+ call(fun call_generic_fallback/3, Config,
+ [pre_init_per_group, Mod, GroupName]);
+init_tc(Mod, {end_per_group, GroupName, _}, Config) ->
+ call(fun call_generic_fallback/3, Config,
+ [pre_end_per_group, Mod, GroupName]);
+init_tc(Mod, {init_per_testcase,TC}, Config) ->
+ call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]);
+init_tc(Mod, {end_per_testcase,TC}, Config) ->
+ call(fun call_generic_fallback/3, Config, [pre_end_per_testcase, Mod, TC]);
+init_tc(Mod, TC = error_in_suite, Config) ->
+ call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]).
%% @doc Called as each test case is completed. This includes all configuration
%% tests.
@@ -126,23 +128,23 @@ end_tc(Mod, init_per_suite, Config, _Result, Return) ->
end_tc(Mod, end_per_suite, Config, Result, _Return) ->
call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config],
'$ct_no_change');
-end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) ->
- call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config],
- '$ct_no_change');
+end_tc(Mod, {init_per_group, GroupName, _}, Config, _Result, Return) ->
+ call(fun call_generic_fallback/3, Return,
+ [post_init_per_group, Mod, GroupName, Config], '$ct_no_change');
end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) ->
- Res = call(fun call_generic/3, Result,
- [post_end_per_group, GroupName, Config], '$ct_no_change'),
+ Res = call(fun call_generic_fallback/3, Result,
+ [post_end_per_group, Mod, GroupName, Config], '$ct_no_change'),
maybe_stop_locker(Mod, GroupName, Properties),
Res;
-end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) ->
- call(fun call_generic/3, Result, [post_init_per_testcase, TC, Config],
- '$ct_no_change');
-end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) ->
- call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config],
- '$ct_no_change');
-end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) ->
- call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config],
- '$ct_no_change').
+end_tc(Mod, {init_per_testcase,TC}, Config, Result, _Return) ->
+ call(fun call_generic_fallback/3, Result,
+ [post_init_per_testcase, Mod, TC, Config], '$ct_no_change');
+end_tc(Mod, {end_per_testcase,TC}, Config, Result, _Return) ->
+ call(fun call_generic_fallback/3, Result,
+ [post_end_per_testcase, Mod, TC, Config], '$ct_no_change');
+end_tc(Mod, TC = error_in_suite, Config, Result, _Return) ->
+ call(fun call_generic_fallback/3, Result,
+ [post_end_per_testcase, Mod, TC, Config], '$ct_no_change').
%% Case = TestCase | {TestCase,GroupName}
@@ -181,15 +183,21 @@ call_terminate(#ct_hook_config{ module = Mod, state = State} = Hook, _, _) ->
{[],Hook}.
call_cleanup(#ct_hook_config{ module = Mod, state = State} = Hook,
- Reason, [Function, _Suite | Args]) ->
+ Reason, [Function | Args]) ->
NewState = catch_apply(Mod,Function, Args ++ [Reason, State],
- State),
+ State, true),
{Reason, Hook#ct_hook_config{ state = NewState } }.
-call_generic(#ct_hook_config{ module = Mod, state = State} = Hook,
- Value, [Function | Args]) ->
+call_generic(Hook, Value, Meta) ->
+ do_call_generic(Hook, Value, Meta, false).
+
+call_generic_fallback(Hook, Value, Meta) ->
+ do_call_generic(Hook, Value, Meta, true).
+
+do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook,
+ Value, [Function | Args], Fallback) ->
{NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State],
- {Value,State}),
+ {Value,State}, Fallback),
{NewValue, Hook#ct_hook_config{ state = NewState } }.
%% Generic call function
@@ -257,15 +265,15 @@ remove(Key,List) when is_list(List) ->
remove(_, Else) ->
Else.
-%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc
-scope([pre_init_per_testcase, TC|_]) ->
- [post_init_per_testcase, TC];
-scope([pre_end_per_testcase, TC|_]) ->
- [post_end_per_testcase, TC];
-scope([pre_init_per_group, GroupName|_]) ->
- [post_end_per_group, GroupName];
-scope([post_init_per_group, GroupName|_]) ->
- [post_end_per_group, GroupName];
+%% Translate scopes, i.e. is_tuplenit_per_group,group1 -> end_per_group,group1 etc
+scope([pre_init_per_testcase, SuiteName, TC|_]) ->
+ [post_init_per_testcase, SuiteName, TC];
+scope([pre_end_per_testcase, SuiteName, TC|_]) ->
+ [post_end_per_testcase, SuiteName, TC];
+scope([pre_init_per_group, SuiteName, GroupName|_]) ->
+ [post_end_per_group, SuiteName, GroupName];
+scope([post_init_per_group, SuiteName, GroupName|_]) ->
+ [post_end_per_group, SuiteName, GroupName];
scope([pre_init_per_suite, SuiteName|_]) ->
[post_end_per_suite, SuiteName];
scope([post_init_per_suite, SuiteName|_]) ->
@@ -273,14 +281,29 @@ scope([post_init_per_suite, SuiteName|_]) ->
scope(init) ->
none.
-terminate_if_scope_ends(HookId, [on_tc_skip,_Suite,{end_per_group,Name}],
+strip_config([post_init_per_testcase, SuiteName, TC|_]) ->
+ [post_init_per_testcase, SuiteName, TC];
+strip_config([post_end_per_testcase, SuiteName, TC|_]) ->
+ [post_end_per_testcase, SuiteName, TC];
+strip_config([post_init_per_group, SuiteName, GroupName|_]) ->
+ [post_init_per_group, SuiteName, GroupName];
+strip_config([post_end_per_group, SuiteName, GroupName|_]) ->
+ [post_end_per_group, SuiteName, GroupName];
+strip_config([post_init_per_suite, SuiteName|_]) ->
+ [post_init_per_suite, SuiteName];
+strip_config([post_end_per_suite, SuiteName|_]) ->
+ [post_end_per_suite, SuiteName];
+strip_config(Other) ->
+ Other.
+
+
+terminate_if_scope_ends(HookId, [on_tc_skip,Suite,{end_per_group,Name}],
Hooks) ->
- terminate_if_scope_ends(HookId, [post_end_per_group, Name], Hooks);
+ terminate_if_scope_ends(HookId, [post_end_per_group, Suite, Name], Hooks);
terminate_if_scope_ends(HookId, [on_tc_skip,Suite,end_per_suite], Hooks) ->
terminate_if_scope_ends(HookId, [post_end_per_suite, Suite], Hooks);
-terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] ->
- terminate_if_scope_ends(HookId,[Function,Tag],Hooks);
-terminate_if_scope_ends(HookId, Function, Hooks) ->
+terminate_if_scope_ends(HookId, Function0, Hooks) ->
+ Function = strip_config(Function0),
case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of
#ct_hook_config{ id = HookId, scope = Function} = Hook ->
terminate([Hook]),
@@ -384,21 +407,29 @@ pos(Id,[_|Rest],Num) ->
catch_apply(M,F,A, Default) ->
+ catch_apply(M,F,A,Default,false).
+catch_apply(M,F,A, Default, Fallback) ->
+ not erlang:module_loaded(M) andalso (catch M:module_info()),
+ case erlang:function_exported(M,F,length(A)) of
+ false when Fallback ->
+ catch_apply(M,F,tl(A),Default,false);
+ false ->
+ Default;
+ true ->
+ catch_apply(M,F,A)
+ end.
+
+catch_apply(M,F,A) ->
try
- erlang:apply(M,F,A)
+ erlang:apply(M,F,A)
catch _:Reason ->
- case erlang:get_stacktrace() of
- %% Return the default if it was the CTH module which did not have the function.
- [{M,F,A,_}|_] when Reason == undef ->
- Default;
- Trace ->
- ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p",
- [error,{Reason,Trace}]),
- throw({error_in_cth_call,
- lists:flatten(
- io_lib:format("~w:~w/~w CTH call failed",
- [M,F,length(A)]))})
- end
+ Trace = erlang:get_stacktrace(),
+ ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p",
+ [error,{Reason,Trace}]),
+ throw({error_in_cth_call,
+ lists:flatten(
+ io_lib:format("~w:~w/~w CTH call failed",
+ [M,F,length(A)]))})
end.
diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl
index 883da0da0a..ce8852b3ea 100644
--- a/lib/common_test/src/cth_conn_log.erl
+++ b/lib/common_test/src/cth_conn_log.erl
@@ -54,8 +54,8 @@
-include_lib("common_test/include/ct.hrl").
-export([init/2,
- pre_init_per_testcase/3,
- post_end_per_testcase/4]).
+ pre_init_per_testcase/4,
+ post_end_per_testcase/5]).
%%----------------------------------------------------------------------
%% Exported types
@@ -104,7 +104,7 @@ get_log_opts(Mod,Opts) ->
Hosts = proplists:get_value(hosts,Opts,[]),
{LogType,Hosts}.
-pre_init_per_testcase(TestCase,Config,CthState) ->
+pre_init_per_testcase(_Suite,TestCase,Config,CthState) ->
Logs =
lists:map(
fun({ConnMod,{LogType,Hosts}}) ->
@@ -158,7 +158,7 @@ pre_init_per_testcase(TestCase,Config,CthState) ->
ct_util:update_testdata(?MODULE, Update, [create]),
{Config,CthState}.
-post_end_per_testcase(TestCase,_Config,Return,CthState) ->
+post_end_per_testcase(_Suite,TestCase,_Config,Return,CthState) ->
Update =
fun(PrevUsers) ->
case lists:delete(TestCase, PrevUsers) of
diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl
index 6d77d7ee9e..eda090d4f5 100644
--- a/lib/common_test/src/cth_log_redirect.erl
+++ b/lib/common_test/src/cth_log_redirect.erl
@@ -28,10 +28,10 @@
%% CTH Callbacks
-export([id/1, init/2,
pre_init_per_suite/3, pre_end_per_suite/3, post_end_per_suite/4,
- pre_init_per_group/3, post_init_per_group/4,
- pre_end_per_group/3, post_end_per_group/4,
- pre_init_per_testcase/3, post_init_per_testcase/4,
- pre_end_per_testcase/3, post_end_per_testcase/4]).
+ pre_init_per_group/4, post_init_per_group/5,
+ pre_end_per_group/4, post_end_per_group/5,
+ pre_init_per_testcase/4, post_init_per_testcase/5,
+ pre_end_per_testcase/4, post_end_per_testcase/5]).
%% Event handler Callbacks
-export([init/1,
@@ -71,11 +71,11 @@ post_end_per_suite(_Suite, Config, Return, State) ->
set_curr_func(undefined, Config),
{Return, State}.
-pre_init_per_group(Group, Config, State) ->
+pre_init_per_group(_Suite, Group, Config, State) ->
set_curr_func({group,Group,init_per_group}, Config),
{Config, State}.
-post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) ->
+post_init_per_group(_Suite, Group, Config, Result, tc_log_async) when is_list(Config) ->
case lists:member(parallel,proplists:get_value(
tc_group_properties,Config,[])) of
true ->
@@ -83,33 +83,33 @@ post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) ->
false ->
{Result, tc_log_async}
end;
-post_init_per_group(_Group, _Config, Result, State) ->
+post_init_per_group(_Suite, _Group, _Config, Result, State) ->
{Result, State}.
-pre_init_per_testcase(TC, Config, State) ->
+pre_init_per_testcase(_Suite, TC, Config, State) ->
set_curr_func(TC, Config),
{Config, State}.
-post_init_per_testcase(_TC, _Config, Return, State) ->
+post_init_per_testcase(_Suite, _TC, _Config, Return, State) ->
{Return, State}.
-pre_end_per_testcase(_TC, Config, State) ->
+pre_end_per_testcase(_Suite, _TC, Config, State) ->
{Config, State}.
-post_end_per_testcase(_TC, _Config, Result, State) ->
+post_end_per_testcase(_Suite, _TC, _Config, Result, State) ->
%% Make sure that the event queue is flushed
%% before ending this test case.
gen_event:call(error_logger, ?MODULE, flush, 300000),
{Result, State}.
-pre_end_per_group(Group, Config, {tc_log, Group}) ->
+pre_end_per_group(_Suite, Group, Config, {tc_log, Group}) ->
set_curr_func({group,Group,end_per_group}, Config),
{Config, set_log_func(tc_log_async)};
-pre_end_per_group(Group, Config, State) ->
+pre_end_per_group(_Suite, Group, Config, State) ->
set_curr_func({group,Group,end_per_group}, Config),
{Config, State}.
-post_end_per_group(_Group, Config, Return, State) ->
+post_end_per_group(_Suite, _Group, Config, Return, State) ->
set_curr_func({group,undefined}, Config),
{Return, State}.
diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl
index 59b916851e..c4941948cc 100644
--- a/lib/common_test/src/cth_surefire.erl
+++ b/lib/common_test/src/cth_surefire.erl
@@ -33,16 +33,16 @@
-export([pre_end_per_suite/3]).
-export([post_end_per_suite/4]).
--export([pre_init_per_group/3]).
--export([post_init_per_group/4]).
--export([pre_end_per_group/3]).
--export([post_end_per_group/4]).
+-export([pre_init_per_group/4]).
+-export([post_init_per_group/5]).
+-export([pre_end_per_group/4]).
+-export([post_end_per_group/5]).
--export([pre_init_per_testcase/3]).
--export([post_end_per_testcase/4]).
+-export([pre_init_per_testcase/4]).
+-export([post_end_per_testcase/5]).
--export([on_tc_fail/3]).
--export([on_tc_skip/3]).
+-export([on_tc_fail/4]).
+-export([on_tc_skip/4]).
-export([terminate/1]).
@@ -116,29 +116,29 @@ pre_end_per_suite(_Suite,Config,State) ->
post_end_per_suite(_Suite,Config,Result,State) ->
{Result, end_tc(end_per_suite,Config,Result,State)}.
-pre_init_per_group(Group,Config,State) ->
+pre_init_per_group(_Suite,Group,Config,State) ->
{Config, init_tc(State#state{ curr_group = [Group|State#state.curr_group]},
Config)}.
-post_init_per_group(_Group,Config,Result,State) ->
+post_init_per_group(_Suite,_Group,Config,Result,State) ->
{Result, end_tc(init_per_group,Config,Result,State)}.
-pre_end_per_group(_Group,Config,State) ->
+pre_end_per_group(_Suite,_Group,Config,State) ->
{Config, init_tc(State, Config)}.
-post_end_per_group(_Group,Config,Result,State) ->
+post_end_per_group(_Suite,_Group,Config,Result,State) ->
NewState = end_tc(end_per_group, Config, Result, State),
{Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}.
-pre_init_per_testcase(_TC,Config,State) ->
+pre_init_per_testcase(_Suite,_TC,Config,State) ->
{Config, init_tc(State, Config)}.
-post_end_per_testcase(TC,Config,Result,State) ->
+post_end_per_testcase(_Suite,TC,Config,Result,State) ->
{Result, end_tc(TC,Config, Result,State)}.
-on_tc_fail(_TC, _Res, State = #state{test_cases = []}) ->
+on_tc_fail(_Suite,_TC, _Res, State = #state{test_cases = []}) ->
State;
-on_tc_fail(_TC, Res, State) ->
+on_tc_fail(_Suite,_TC, Res, State) ->
TCs = State#state.test_cases,
TC = hd(TCs),
NewTC = TC#testcase{
@@ -146,10 +146,9 @@ on_tc_fail(_TC, Res, State) ->
{fail,lists:flatten(io_lib:format("~p",[Res]))} },
State#state{ test_cases = [NewTC | tl(TCs)]}.
-on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0)
- when Type == tc_auto_skip; Type == tc_user_skip ->
- on_tc_skip(ConfigFunc, Res, State0);
-on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->
+on_tc_skip(Suite,{ConfigFunc,_GrName}, Res, State) ->
+ on_tc_skip(Suite,ConfigFunc, Res, State);
+on_tc_skip(Suite,Tc, Res, State0) ->
TcStr = atom_to_list(Tc),
State =
case State0#state.test_cases of
@@ -158,11 +157,7 @@ on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->
_ ->
State0
end,
- do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[])));
-on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) ->
- State;
-on_tc_skip(_Tc, Res, State) ->
- do_tc_skip(Res, State).
+ do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(set_suite(Suite,State),[]))).
do_tc_skip(Res, State) ->
TCs = State#state.test_cases,
@@ -209,6 +204,12 @@ end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite,
result = passed }|
State#state.test_cases],
tc_log = ""}. % so old tc_log is not set if next is on_tc_skip
+
+set_suite(Suite,#state{curr_suite=undefined}=State) ->
+ State#state{curr_suite=Suite, curr_suite_ts=?now};
+set_suite(_,State) ->
+ State.
+
close_suite(#state{ test_cases = [] } = State) ->
State;
close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) ->
@@ -228,7 +229,8 @@ close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) ->
testcases = lists:reverse(TCs),
log = SuiteLog,
url = SuiteUrl},
- State#state{ test_cases = [],
+ State#state{ curr_suite = undefined,
+ test_cases = [],
test_suites = [Suite | State#state.test_suites]}.
terminate(State = #state{ test_cases = [] }) ->
diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index 924086f2bd..be49191f2e 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -778,9 +778,9 @@ spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid,
%% if init_per_testcase fails, the test case
%% should be skipped
try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why),
- do_init_tc_call(Mod,{end_per_testcase,Func},
+ do_init_tc_call(Mod,{end_per_testcase_not_run,Func},
[CurrConf],{ok,[CurrConf]}),
- do_end_tc_call(Mod,{end_per_testcase,Func},
+ do_end_tc_call(Mod,{end_per_testcase_not_run,Func},
{Pid,Skip,[CurrConf]}, Why) end of
_ -> ok
catch
@@ -1151,14 +1151,14 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->
Args
end,
EPTCInitRes =
- case do_init_tc_call(Mod,{end_per_testcase,Func},
+ case do_init_tc_call(Mod,{end_per_testcase_not_run,Func},
IPTCEndRes,Return) of
{ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->
{Return,EPTCInitConfig};
_ ->
- Return
+ {Return,IPTCEndRes}
end,
- do_end_tc_call1(Mod, {end_per_testcase,Func},
+ do_end_tc_call1(Mod, {end_per_testcase_not_run,Func},
EPTCInitRes, Return);
_Ok ->
do_end_tc_call1(Mod, IPTC, Res, Return)
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index b52e4bef9b..39c523f8b3 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -2051,17 +2051,21 @@ add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) ->
add_init_and_end_per_suite([], LastMod, LastRef, FwMod) ->
%% we'll add end_per_suite here even if it's not exported
%% (and simply let the call fail if it's missing)
- case erlang:function_exported(LastMod, end_per_suite, 1) of
- true ->
- [{conf,LastRef,[],{LastMod,end_per_suite}}];
- false ->
+ case {erlang:function_exported(LastMod, end_per_suite, 1),
+ erlang:function_exported(LastMod, init_per_suite, 1)} of
+ {false,false} ->
%% let's call a "fake" end_per_suite if it exists
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
[{conf,LastRef,[{suite,LastMod}],{FwMod,end_per_suite}}];
false ->
[{conf,LastRef,[],{LastMod,end_per_suite}}]
- end
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ [{conf,LastRef,[],{LastMod,end_per_suite}}]
end.
do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
@@ -2070,11 +2074,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
_ -> ok
end,
{Init,NextMod,NextRef} =
- case erlang:function_exported(Mod, init_per_suite, 1) of
- true ->
- Ref = make_ref(),
- {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref};
- false ->
+ case {erlang:function_exported(Mod, init_per_suite, 1),
+ erlang:function_exported(Mod, end_per_suite, 1)} of
+ {false,false} ->
%% let's call a "fake" init_per_suite if it exists
case erlang:function_exported(FwMod, init_per_suite, 1) of
true ->
@@ -2083,8 +2085,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
{FwMod,init_per_suite}}],Mod,Ref};
false ->
{[],Mod,undefined}
- end
-
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ Ref = make_ref(),
+ {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref}
end,
Cases =
if LastRef==undefined ->
@@ -2094,10 +2101,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
true ->
%% we'll add end_per_suite here even if it's not exported
%% (and simply let the call fail if it's missing)
- case erlang:function_exported(LastMod, end_per_suite, 1) of
- true ->
- [{conf,LastRef,[],{LastMod,end_per_suite}}|Init];
- false ->
+ case {erlang:function_exported(LastMod, end_per_suite, 1),
+ erlang:function_exported(LastMod, init_per_suite, 1)} of
+ {false,false} ->
%% let's call a "fake" end_per_suite if it exists
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
@@ -2105,8 +2111,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
{FwMod,end_per_suite}}|Init];
false ->
[{conf,LastRef,[],{LastMod,end_per_suite}}|Init]
- end
- end
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ [{conf,LastRef,[],{LastMod,end_per_suite}}|Init]
+ end
end,
{Cases,NextMod,NextRef}.
@@ -2115,11 +2126,9 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) ->
No when No==undefined ; No==skipped_suite ->
{[],Mod,skipped_suite};
_Ref ->
- case erlang:function_exported(LastMod, end_per_suite, 1) of
- true ->
- {[{conf,LastRef,[],{LastMod,end_per_suite}}],
- Mod,skipped_suite};
- false ->
+ case {erlang:function_exported(LastMod, end_per_suite, 1),
+ erlang:function_exported(LastMod, init_per_suite, 1)} of
+ {false,false} ->
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
%% let's call "fake" end_per_suite if it exists
@@ -2128,7 +2137,13 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) ->
false ->
{[{conf,LastRef,[],{LastMod,end_per_suite}}],
Mod,skipped_suite}
- end
+ end;
+ _ ->
+ %% If any of these exist, the other should too
+ %% (required and documented). If it isn't, it will fail
+ %% with reason 'undef'.
+ {[{conf,LastRef,[],{LastMod,end_per_suite}}],
+ Mod,skipped_suite}
end
end.
@@ -2924,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
exit(framework_error);
%% sequential execution of test case finished
{Time,RetVal,_} ->
+ RetTag =
+ if is_tuple(RetVal) -> element(1,RetVal);
+ true -> undefined
+ end,
{Failed,Status1} =
- case Time of
- died ->
- {true,update_status(failed, Mod, Func, Status)};
- _ when is_tuple(RetVal) ->
- case element(1, RetVal) of
- R when R=='EXIT'; R==failed ->
- {true,update_status(failed, Mod, Func, Status)};
- R when R==skip; R==skipped ->
- {false,update_status(skipped, Mod, Func, Status)};
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end;
- _ ->
- {false,update_status(ok, Mod, Func, Status)}
- end,
+ case RetTag of
+ Skip when Skip==skip; Skip==skipped ->
+ {false,update_status(skipped, Mod, Func, Status)};
+ Fail when Fail=='EXIT'; Fail==failed ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ when Time==died, RetVal=/=ok ->
+ {true,update_status(failed, Mod, Func, Status)};
+ _ ->
+ {false,update_status(ok, Mod, Func, Status)}
+ end,
case check_prop(sequence, Mode) of
false ->
stop_minor_log_file(),
@@ -3794,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
{died,{timetrap_timeout,TimetrapTimeout}} ->
progress(failed, Num, Mod, Func, GrName, Loc,
timetrap_timeout, TimetrapTimeout, Comment, Style);
- {died,Reason} ->
+ {died,{Skip,Reason}} when Skip==skip; Skip==skipped ->
+ %% died in init_per_testcase
+ progress(skip, Num, Mod, Func, GrName, Loc, Reason,
+ Time, Comment, Style);
+ {died,Reason} when Reason=/=ok ->
+ %% (If Reason==ok it means that process died in
+ %% end_per_testcase after successfully completing the
+ %% test case itself - then we shall not fail, but a
+ %% warning will be issued in the comment field.)
progress(failed, Num, Mod, Func, GrName, Loc, Reason,
Time, Comment, Style);
{_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
@@ -3943,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
[get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
{ReportTag,Reason1}}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
ReasonStr = escape_chars(reason_to_string(Reason1)),
ReasonStr1 = lists:flatten([string:strip(S,left) ||
S <- string:tokens(ReasonStr,[$\n])]),
@@ -3957,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
_ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")"
end,
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"~ts\">SKIPPED</font></td>"
"<td>~ts~ts</td></tr>\n",
- [Time,Color,ReasonStr2,Comment1]),
+ [TimeStr,Color,ReasonStr2,Comment1]),
FormatLoc = test_server_sup:format_loc(Loc),
print(minor, "=== Location: ~ts", [FormatLoc]),
print(minor, "=== Reason: ~ts", [ReasonStr1]),
@@ -4098,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
Comment0, {St0,St1}) ->
print(minor, "successfully completed test case", []),
test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]),
+ TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+ true -> "~w"
+ end, [Time]),
Comment =
case RetVal of
{comment,RetComment} ->
@@ -4116,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
end,
print(major, "=elapsed ~p", [Time]),
print(html,
- "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+ "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
"<td><font color=\"green\">Ok</font></td>"
"~ts</tr>\n",
- [Time,Comment]),
+ [TimeStr,Comment]),
print(minor,
escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])),
[]),
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index fae23484e6..621f3b6d2d 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -1531,17 +1531,17 @@ test_events(config_func_errors) ->
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}},
- {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,test_stats,{0,0,{0,1}}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}},
- {?eh,test_stats,{0,2,{0,0}}},
+ {?eh,test_stats,{1,0,{0,1}}},
[{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}},
- {?eh,test_stats,{0,3,{0,0}}},
+ {?eh,test_stats,{1,0,{0,2}}},
{?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}],
@@ -1549,7 +1549,7 @@ test_events(config_func_errors) ->
{?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}},
{?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},
{?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}},
- {?eh,test_stats,{0,4,{0,0}}},
+ {?eh,test_stats,{2,0,{0,2}}},
{?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}},
{?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}],
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index bc716fb5e3..93bcb8fe52 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -82,10 +82,13 @@ all(suite) ->
scope_suite_state_cth,
fail_pre_suite_cth, double_fail_pre_suite_cth,
fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
+ skip_pre_init_tc_cth,
skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
state_update_cth, options_cth, same_id_cth,
fail_n_skip_with_minimal_cth, prio_cth, no_config,
- data_dir, cth_log
+ no_init_suite_config, no_init_config, no_end_config,
+ failed_sequence, repeat_force_stop, config_clash,
+ callbacks_on_skip, fallback, data_dir, cth_log
]
).
@@ -190,6 +193,10 @@ skip_post_suite_cth(Config) when is_list(Config) ->
do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl",
[skip_post_suite_cth],Config).
+skip_pre_init_tc_cth(Config) ->
+ do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl",
+ [skip_pre_init_tc_cth],Config).
+
recover_post_suite_cth(Config) when is_list(Config) ->
do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl",
[recover_post_suite_cth],Config).
@@ -223,6 +230,16 @@ no_config(Config) when is_list(Config) ->
do_test(no_config, "ct_no_config_SUITE.erl",
[verify_config_cth],Config).
+no_init_suite_config(Config) when is_list(Config) ->
+ do_test(no_init_suite_config, "ct_no_init_suite_config_SUITE.erl",
+ [empty_cth],Config).
+
+no_init_config(Config) when is_list(Config) ->
+ do_test(no_init_config, "ct_no_init_config_SUITE.erl",[empty_cth],Config).
+
+no_end_config(Config) when is_list(Config) ->
+ do_test(no_end_config, "ct_no_end_config_SUITE.erl",[empty_cth],Config).
+
data_dir(Config) when is_list(Config) ->
do_test(data_dir, "ct_data_dir_SUITE.erl",
[verify_data_dir_cth],Config).
@@ -254,24 +271,53 @@ cth_log(Config) when is_list(Config) ->
end, UnexpIoLogs),
ok.
+%% OTP-10599 adds the Suite argument as first argument to all hook
+%% callbacks that did not have a Suite argument from before. This test
+%% checks that ct_hooks will fall back to old versions of callbacks if
+%% new versions are not exported.
+fallback(Config) ->
+ do_test(fallback, "all_hook_callbacks_SUITE.erl",[fallback_cth], Config).
+
+%% Test that expected callbacks, and only those, are called when tests
+%% are skipped in different ways
+callbacks_on_skip(Config) ->
+ do_test(callbacks_on_skip, {spec,"skip.spec"},[skip_cth], Config).
+
+%% Test that expected callbacks, and only those, are called when tests
+%% are skipped due to failed sequence
+failed_sequence(Config) ->
+ do_test(failed_sequence, "seq_SUITE.erl", [skip_cth], Config).
+
+%% Test that expected callbacks, and only those, are called when tests
+%% are skipped due to {force_stop,skip_rest} option
+repeat_force_stop(Config) ->
+ do_test(repeat_force_stop, "repeat_SUITE.erl", [skip_cth], Config, ok, 2,
+ [{force_stop,skip_rest},{duration,"000009"}]).
+
+%% Test that expected callbacks, and only those, are called when a test
+%% are fails due to clash in config alias names
+config_clash(Config) ->
+ do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config).
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
-do_test(Tag, SWC, CTHs, Config) ->
- do_test(Tag, SWC, CTHs, Config, ok).
-do_test(Tag, SWC, CTHs, Config, {error,_} = Res) ->
- do_test(Tag, SWC, CTHs, Config, Res, 1);
-do_test(Tag, SWC, CTHs, Config, Res) ->
- do_test(Tag, SWC, CTHs, Config, Res, 2).
+do_test(Tag, WTT, CTHs, Config) ->
+ do_test(Tag, WTT, CTHs, Config, ok).
+do_test(Tag, WTT, CTHs, Config, {error,_} = Res) ->
+ do_test(Tag, WTT, CTHs, Config, Res, 1,[]);
+do_test(Tag, WTT, CTHs, Config, Res) ->
+ do_test(Tag, WTT, CTHs, Config, Res, 2,[]).
-do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) ->
+do_test(Tag, WhatToTest, CTHs, Config, Res, EC, ExtraOpts) when is_list(WhatToTest) ->
+ do_test(Tag, {suite,WhatToTest}, CTHs, Config, Res, EC, ExtraOpts);
+do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) ->
DataDir = ?config(data_dir, Config),
- Suites = filelib:wildcard(
- filename:join([DataDir,"cth/tests",SuiteWildCard])),
- {Opts,ERPid} = setup([{suite,Suites},
- {ct_hooks,CTHs},{label,Tag}], Config),
+ Files = filelib:wildcard(
+ filename:join([DataDir,"cth/tests",Wildcard])),
+ {Opts,ERPid} =
+ setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config),
Res = ct_test_support:run(Opts, Config),
Events = ct_test_support:get_events(ERPid, Config),
@@ -323,10 +369,10 @@ test_events(one_empty_cth) ->
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
- {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{empty_cth,post_init_per_testcase,[test_case,'$proplist','_',[]]}},
- {?eh,cth,{empty_cth,pre_end_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
@@ -355,10 +401,10 @@ test_events(two_empty_cth) ->
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
@@ -423,8 +469,8 @@ test_events(minimal_and_maximal_cth) ->
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
@@ -512,8 +558,8 @@ test_events(scope_per_suite_cth) ->
{?eh,tc_done,{ct_scope_per_suite_cth_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_scope_per_suite_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}},
@@ -538,8 +584,8 @@ test_events(scope_suite_cth) ->
{?eh,tc_done,{ct_scope_suite_cth_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_scope_suite_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}},
@@ -561,17 +607,17 @@ test_events(scope_per_group_cth) ->
[{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}},
{?eh,cth,{'_',id,[[]]}},
{?eh,cth,{'_',init,['_',[]]}},
- {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}},
+ {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1, '$proplist','$proplist',[]]}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}},
{?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}},
- {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}},
+ {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}},
{?eh,cth,{'_',terminate,[[]]}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}],
@@ -592,8 +638,8 @@ test_events(scope_per_suite_state_cth) ->
{?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',[test]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}},
{?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}},
@@ -618,8 +664,8 @@ test_events(scope_suite_state_cth) ->
{?eh,tc_done,{ct_scope_suite_state_cth_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_scope_suite_state_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',[test]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}},
{?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}},
@@ -641,17 +687,17 @@ test_events(scope_per_group_state_cth) ->
[{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]}}},
{?eh,cth,{'_',id,[[test]]}},
{?eh,cth,{'_',init,['_',[test]]}},
- {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}},
+ {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','$proplist',[test]]}},
{?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}},
{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',[test]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',ok,[test]]}},
{?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}},
- {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}},
- {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}},
+ {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist',[test]]}},
+ {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','_',[test]]}},
{?eh,cth,{'_',terminate,[[test]]}},
{?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}],
@@ -674,14 +720,14 @@ test_events(fail_pre_suite_cth) ->
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,
{failed, {error,"Test failure"}}}},
{?eh,cth,{'_',on_tc_fail,
- [init_per_suite,{failed,"Test failure"},[]]}},
+ [ct_cth_empty_SUITE,init_per_suite,"Test failure",[]]}},
{?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,
{failed,{ct_cth_empty_SUITE,init_per_suite,
{failed,"Test failure"}}}}},
{?eh,cth,{'_',on_tc_skip,
- [test_case, {tc_auto_skip,
+ [ct_cth_empty_SUITE,test_case, {tc_auto_skip,
{failed, {ct_cth_empty_SUITE, init_per_suite,
{failed, "Test failure"}}}},[]]}},
@@ -690,7 +736,7 @@ test_events(fail_pre_suite_cth) ->
{failed, {ct_cth_empty_SUITE, init_per_suite,
{failed, "Test failure"}}}}},
{?eh,cth,{'_',on_tc_skip,
- [end_per_suite, {tc_auto_skip,
+ [ct_cth_empty_SUITE,end_per_suite, {tc_auto_skip,
{failed, {ct_cth_empty_SUITE, init_per_suite,
{failed, "Test failure"}}}},[]]}},
@@ -727,17 +773,17 @@ test_events(fail_post_suite_cth) ->
{?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,
{failed,{error,"Test failure"}}}},
- {?eh,cth,{'_',on_tc_fail,[init_per_suite, {failed,"Test failure"}, []]}},
+ {?eh,cth,{'_',on_tc_fail,[ct_cth_empty_SUITE,init_per_suite, "Test failure", []]}},
{?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,
{failed,{ct_cth_empty_SUITE,init_per_suite,
{failed,"Test failure"}}}}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}},
+ {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_auto_skip,'_'},[]]}},
{?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,
{failed, {ct_cth_empty_SUITE, init_per_suite,
{failed, "Test failure"}}}}},
- {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,'_'},[]]}},
+ {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,end_per_suite,{tc_auto_skip,'_'},[]]}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,cth, {'_',terminate,[[]]}},
@@ -754,10 +800,10 @@ test_events(skip_pre_suite_cth) ->
{?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist',{skip,"Test skip"},[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}},
{?eh,cth,{'_',on_tc_skip,
- [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}},
+ [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}},
{?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}},
+ {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}},
{?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
@@ -776,27 +822,29 @@ test_events(skip_pre_end_cth) ->
[{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}},
{?eh,cth,{'_',id,[[]]}},
{?eh,cth,{'_',init,['_',[]]}},
- {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}},
+ {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','$proplist',[]]}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}},
{?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}},
{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}},
- {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}},
+ {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},
{skipped,"Test skip"}}}],
- {?eh,cth,{'_',on_tc_skip,[{end_per_group,group1},
- {tc_user_skip,{skipped,"Test skip"}},
+ {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE,
+ {end_per_group,group1},
+ {tc_user_skip,"Test skip"},
[]]}},
{?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}},
{?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite,
{skipped,"Test skip"}}},
- {?eh,cth,{'_',on_tc_skip,[end_per_suite,
- {tc_user_skip,{skipped,"Test skip"}},
+ {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE,
+ end_per_suite,
+ {tc_user_skip,"Test skip"},
[]]}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,cth,{'_',terminate,[[]]}},
@@ -814,10 +862,10 @@ test_events(skip_post_suite_cth) ->
{?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}},
{?eh,cth,{'_',on_tc_skip,
- [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}},
+ [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}},
{?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}},
- {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}},
+ {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}},
{?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}},
@@ -826,6 +874,41 @@ test_events(skip_post_suite_cth) ->
{?eh,stop_logging,[]}
];
+test_events(skip_pre_init_tc_cth) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,init,['_',[]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [ct_cth_empty_SUITE,test_case,'$proplist',
+ {skip,"Skipped in pre_init_per_testcase"},
+ []]}},
+ {?eh,tc_done,{ct_cth_empty_SUITE,test_case,
+ {skipped,"Skipped in pre_init_per_testcase"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [ct_cth_empty_SUITE,test_case,
+ {tc_user_skip,"Skipped in pre_init_per_testcase"},
+ []]}},
+ {?eh,test_stats,{0,0,{1,0}}},
+ {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [ct_cth_empty_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(recover_post_suite_cth) ->
Suite = ct_cth_fail_per_suite_SUITE,
[
@@ -840,9 +923,9 @@ test_events(recover_post_suite_cth) ->
{?eh,tc_start,{Suite,test_case}},
{?eh,cth,{'_',pre_init_per_testcase,
- [test_case, not_contains([tc_status]),[]]}},
+ [Suite,test_case, not_contains([tc_status]),[]]}},
{?eh,cth,{'_',post_end_per_testcase,
- [test_case, contains([tc_status]),'_',[]]}},
+ [Suite,test_case, contains([tc_status]),'_',[]]}},
{?eh,tc_done,{Suite,test_case,ok}},
{?eh,tc_start,{Suite,end_per_suite}},
@@ -876,13 +959,15 @@ test_events(update_config_cth) ->
{?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}},
{?eh,cth,{'_',pre_init_per_group,
- [group1,contains(
+ [ct_update_config_SUITE,
+ group1,contains(
[post_init_per_suite,
init_per_suite,
pre_init_per_suite]),
[]]}},
{?eh,cth,{'_',post_init_per_group,
- [group1,
+ [ct_update_config_SUITE,
+ group1,
contains(
[post_init_per_suite,
init_per_suite,
@@ -898,7 +983,8 @@ test_events(update_config_cth) ->
{?eh,tc_start,{ct_update_config_SUITE,test_case}},
{?eh,cth,{'_',pre_init_per_testcase,
- [test_case,contains(
+ [ct_update_config_SUITE,
+ test_case,contains(
[post_init_per_group,
init_per_group,
pre_init_per_group,
@@ -907,7 +993,8 @@ test_events(update_config_cth) ->
pre_init_per_suite]),
[]]}},
{?eh,cth,{'_',post_end_per_testcase,
- [test_case,contains(
+ [ct_update_config_SUITE,
+ test_case,contains(
[init_per_testcase,
pre_init_per_testcase,
post_init_per_group,
@@ -921,7 +1008,8 @@ test_events(update_config_cth) ->
{?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}},
{?eh,cth,{'_',pre_end_per_group,
- [group1,contains(
+ [ct_update_config_SUITE,
+ group1,contains(
[post_init_per_group,
init_per_group,
pre_init_per_group,
@@ -930,7 +1018,8 @@ test_events(update_config_cth) ->
pre_init_per_suite]),
[]]}},
{?eh,cth,{'_',post_end_per_group,
- [group1,
+ [ct_update_config_SUITE,
+ group1,
contains(
[pre_end_per_group,
post_init_per_group,
@@ -1018,8 +1107,8 @@ test_events(options_cth) ->
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
- {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}},
- {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[test]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[test]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},
{?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
@@ -1051,12 +1140,12 @@ test_events(same_id_cth) ->
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}},
{?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
{negative,
- {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}},
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}},
+ {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}},
{negative,
- {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}},
+ {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}},
{?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}},
@@ -1094,11 +1183,13 @@ test_events(fail_n_skip_with_minimal_cth) ->
{?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case2,{skipped,"skip it"}}},
{?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,test_case3}},
{?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case3,{skipped,"skip it"}}},
- {?eh,cth,{empty_cth,on_tc_skip,[{test_case2,group2},
- {tc_user_skip,{skipped,"skip it"}},
+ {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE,
+ {test_case2,group2},
+ {tc_user_skip,"skip it"},
[]]}},
- {?eh,cth,{empty_cth,on_tc_skip,[{test_case3,group2},
- {tc_user_skip,{skipped,"skip it"}},
+ {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE,
+ {test_case3,group2},
+ {tc_user_skip,"skip it"},
[]]}},
{?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,{end_per_group,
group2,[parallel]}}},
@@ -1115,13 +1206,24 @@ test_events(fail_n_skip_with_minimal_cth) ->
];
test_events(prio_cth) ->
- GenPre = fun(Func,States) ->
- [{?eh,cth,{'_',Func,['_','_',State]}} || State <- States]
+ GenPre = fun(Func,States) when Func==pre_init_per_suite;
+ Func==pre_end_per_suite ->
+ [{?eh,cth,{'_',Func,['_','_',State]}} ||
+ State <- States];
+ (Func,States) ->
+ [{?eh,cth,{'_',Func,['_','_','_',State]}} ||
+ State <- States]
end,
- GenPost = fun(Func,States) ->
- [{?eh,cth,{'_',Func,['_','_','_',State]}} || State <- States]
- end,
+ GenPost = fun(Func,States) when Func==post_init_per_suite;
+ Func==post_end_per_suite ->
+ [{?eh,cth,{'_',Func,['_','_','_',State]}} ||
+ State <- States];
+ (Func,States) ->
+ [{?eh,cth,{'_',Func,['_','_','_','_',State]}} ||
+ State <- States]
+
+ end,
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}] ++
@@ -1197,30 +1299,30 @@ test_events(no_config) ->
{?eh,tc_done,{ct_framework,init_per_suite,ok}},
{?eh,tc_start,{ct_no_config_SUITE,test_case_1}},
{?eh,cth,{empty_cth,pre_init_per_testcase,
- [test_case_1,'$proplist',[]]}},
+ [ct_no_config_SUITE,test_case_1,'$proplist',[]]}},
{?eh,cth,{empty_cth,post_end_per_testcase,
- [test_case_1,'$proplist',ok,[]]}},
+ [ct_no_config_SUITE,test_case_1,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_no_config_SUITE,test_case_1,ok}},
{?eh,test_stats,{1,0,{0,0}}},
[{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}},
{?eh,cth,{empty_cth,pre_init_per_group,
- [test_group,'$proplist',[]]}},
+ [ct_no_config_SUITE,test_group,'$proplist',[]]}},
{?eh,cth,{empty_cth,post_init_per_group,
- [test_group,'$proplist','$proplist',[]]}},
+ [ct_no_config_SUITE,test_group,'$proplist','$proplist',[]]}},
{?eh,tc_done,{ct_framework,
{init_per_group,test_group,'$proplist'},ok}},
{?eh,tc_start,{ct_no_config_SUITE,test_case_2}},
{?eh,cth,{empty_cth,pre_init_per_testcase,
- [test_case_2,'$proplist',[]]}},
+ [ct_no_config_SUITE,test_case_2,'$proplist',[]]}},
{?eh,cth,{empty_cth,post_end_per_testcase,
- [test_case_2,'$proplist',ok,[]]}},
+ [ct_no_config_SUITE,test_case_2,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_no_config_SUITE,test_case_2,ok}},
{?eh,test_stats,{2,0,{0,0}}},
{?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}},
{?eh,cth,{empty_cth,pre_end_per_group,
- [test_group,'$proplist',[]]}},
+ [ct_no_config_SUITE,test_group,'$proplist',[]]}},
{?eh,cth,{empty_cth,post_end_per_group,
- [test_group,'$proplist',ok,[]]}},
+ [ct_no_config_SUITE,test_group,'$proplist',ok,[]]}},
{?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}],
{?eh,tc_start,{ct_framework,end_per_suite}},
{?eh,cth,{empty_cth,pre_end_per_suite,
@@ -1233,6 +1335,166 @@ test_events(no_config) ->
{?eh,stop_logging,[]}
];
+test_events(no_init_suite_config) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,tc_start,{ct_no_init_suite_config_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [ct_no_init_suite_config_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [ct_no_init_suite_config_SUITE,'$proplist','_',[]]}},
+ {?eh,tc_done,{ct_no_init_suite_config_SUITE,init_per_suite,
+ {failed,{error,{undef,'_'}}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_suite_config_SUITE,
+ init_per_suite,
+ {undef,'_'},[]]}},
+ {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,test_case,
+ {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+ {'EXIT',{undef,'_'}}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [ct_no_init_suite_config_SUITE,
+ test_case,
+ {tc_auto_skip,
+ {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+ {'EXIT',{undef,'_'}}}}},
+ []]}},
+ {?eh,test_stats,{0,0,{0,1}}},
+ {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,end_per_suite,
+ {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+ {'EXIT',{undef,'_'}}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [ct_no_init_suite_config_SUITE,
+ end_per_suite,
+ {tc_auto_skip,
+ {failed,{ct_no_init_suite_config_SUITE,init_per_suite,
+ {'EXIT',{undef,'_'}}}}},
+ []]}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(no_init_config) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,tc_start,{ct_no_init_config_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [ct_no_init_config_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [ct_no_init_config_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_no_init_config_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{ct_no_init_config_SUITE,test_case_1}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [ct_no_init_config_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [ct_no_init_config_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_no_init_config_SUITE,test_case_1,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ [{?eh,tc_start,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}}},
+ {?eh,cth,{empty_cth,pre_init_per_group,
+ [ct_no_init_config_SUITE,test_group,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_group,
+ [ct_no_init_config_SUITE,test_group,'$proplist','_',[]]}},
+ {?eh,tc_done,{ct_no_init_config_SUITE,{init_per_group,test_group,[]},
+ {failed,{error,{undef,'_'}}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_config_SUITE,
+ {init_per_group,test_group},
+ {undef,'_'},[]]}},
+ {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{test_case_2,test_group},
+ {failed,{ct_no_init_config_SUITE,init_per_group,
+ {'EXIT',{undef,'_'}}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE,
+ {test_case_2,test_group},
+ {tc_auto_skip,
+ {failed,
+ {ct_no_init_config_SUITE,init_per_group,
+ {'EXIT',{undef,'_'}}}}},
+ []]}},
+ {?eh,test_stats,{1,0,{0,1}}},
+ {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{end_per_group,test_group},
+ {failed,{ct_no_init_config_SUITE,init_per_group,
+ {'EXIT',{undef,'_'}}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE,
+ {end_per_group,test_group},
+ {tc_auto_skip,
+ {failed,
+ {ct_no_init_config_SUITE,init_per_group,
+ {'EXIT',{undef,'_'}}}}},
+ []]}}],
+ {?eh,tc_start,{ct_no_init_config_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [ct_no_init_config_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [ct_no_init_config_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_no_init_config_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(no_end_config) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,tc_start,{ct_no_end_config_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [ct_no_end_config_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [ct_no_end_config_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_no_end_config_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{ct_no_end_config_SUITE,test_case_1}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [ct_no_end_config_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [ct_no_end_config_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_no_end_config_SUITE,test_case_1,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ [{?eh,tc_start,{ct_no_end_config_SUITE,
+ {init_per_group,test_group,'$proplist'}}},
+ {?eh,cth,{empty_cth,pre_init_per_group,
+ [ct_no_end_config_SUITE,test_group,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_group,
+ [ct_no_end_config_SUITE,test_group,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_no_end_config_SUITE,
+ {init_per_group,test_group,'$proplist'},ok}},
+ {?eh,tc_start,{ct_no_end_config_SUITE,test_case_2}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [ct_no_end_config_SUITE,test_case_2,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [ct_no_end_config_SUITE,test_case_2,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_no_end_config_SUITE,test_case_2,ok}},
+ {?eh,test_stats,{2,0,{0,0}}},
+ {?eh,tc_start,{ct_no_end_config_SUITE,
+ {end_per_group,test_group,'$proplist'}}},
+ {?eh,cth,{empty_cth,pre_end_per_group,
+ [ct_no_end_config_SUITE,test_group,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_group,
+ [ct_no_end_config_SUITE,test_group,'$proplist','_',[]]}},
+ {?eh,tc_done,{ct_no_end_config_SUITE,{end_per_group,test_group,[]},
+ {failed,{error,{undef,'_'}}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,[ct_no_end_config_SUITE,
+ {end_per_group,test_group},
+ {undef,'_'},[]]}}],
+ {?eh,tc_start,{ct_no_end_config_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [ct_no_end_config_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [ct_no_end_config_SUITE,'$proplist','_',[]]}},
+ {?eh,tc_done,{ct_no_end_config_SUITE,end_per_suite,
+ {failed,{error,{undef,'_'}}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(data_dir) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
@@ -1247,30 +1509,30 @@ test_events(data_dir) ->
{?eh,tc_done,{ct_framework,init_per_suite,ok}},
{?eh,tc_start,{ct_data_dir_SUITE,test_case_1}},
{?eh,cth,{empty_cth,pre_init_per_testcase,
- [test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,cth,{empty_cth,post_end_per_testcase,
- [test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,tc_done,{ct_data_dir_SUITE,test_case_1,ok}},
{?eh,test_stats,{1,0,{0,0}}},
[{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}},
{?eh,cth,{empty_cth,pre_init_per_group,
- [test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,cth,{empty_cth,post_init_per_group,
- [test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,tc_done,{ct_framework,
{init_per_group,test_group,'$proplist'},ok}},
{?eh,tc_start,{ct_data_dir_SUITE,test_case_2}},
{?eh,cth,{empty_cth,pre_init_per_testcase,
- [test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,cth,{empty_cth,post_end_per_testcase,
- [test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,tc_done,{ct_data_dir_SUITE,test_case_2,ok}},
{?eh,test_stats,{2,0,{0,0}}},
{?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}},
{?eh,cth,{empty_cth,pre_end_per_group,
- [test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,cth,{empty_cth,post_end_per_group,
- [test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
+ [ct_data_dir_SUITE,test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},
{?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}],
{?eh,tc_start,{ct_framework,end_per_suite}},
{?eh,cth,{empty_cth,pre_end_per_suite,
@@ -1303,6 +1565,645 @@ test_events(cth_log) ->
{?eh,stop_logging,[]}
];
+test_events(fallback) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,tc_start,{all_hook_callbacks_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [all_hook_callbacks_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [all_hook_callbacks_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{all_hook_callbacks_SUITE,init_per_suite,ok}},
+
+ [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}},
+ {?eh,cth,{empty_cth,pre_init_per_group,
+ [fallback_nosuite,test_group,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_group,
+ [fallback_nosuite,test_group,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,
+ {init_per_group,test_group,'$proplist'},ok}},
+ {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [fallback_nosuite,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [fallback_nosuite,test_case,'$proplist',ok,[]]}},
+ {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}},
+ {?eh,cth,{empty_cth,pre_end_per_group,
+ [fallback_nosuite,test_group,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_group,
+ [fallback_nosuite,test_group,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}],
+ {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [fallback_nosuite,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [fallback_nosuite,test_case,'$proplist','_',[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [fallback_nosuite,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [fallback_nosuite,test_case,'$proplist','_',[]]}},
+ {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}},
+ {?eh,test_stats,{2,0,{0,0}}},
+ {?eh,tc_start,{all_hook_callbacks_SUITE,skip_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [fallback_nosuite,skip_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [fallback_nosuite,skip_case,'$proplist',
+ {skip,"Skipped in init_per_testcase/2"},[]]}},
+ {?eh,tc_done,{all_hook_callbacks_SUITE,skip_case,
+ {skipped,"Skipped in init_per_testcase/2"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [fallback_nosuite,skip_case,
+ {tc_user_skip,"Skipped in init_per_testcase/2"},
+ []]}},
+ {?eh,test_stats,{2,0,{1,0}}},
+ {?eh,tc_start,{all_hook_callbacks_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [all_hook_callbacks_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [all_hook_callbacks_SUITE,'$proplist','_',[]]}},
+ {?eh,tc_done,{all_hook_callbacks_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(callbacks_on_skip) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events =
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{6,6,15}},
+
+ %% all_hook_callbacks_SUITE is skipped in spec
+ %% Only the on_tc_skip callback shall be called
+ {?eh,tc_user_skip,{all_hook_callbacks_SUITE,all,"Skipped in spec"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [all_hook_callbacks_SUITE,all,
+ {tc_user_skip,"Skipped in spec"},
+ []]}},
+ {?eh,test_stats,{0,0,{1,0}}},
+
+ %% skip_init_SUITE is skipped in its init_per_suite function
+ %% No group- or testcase-functions shall be called.
+ {?eh,tc_start,{skip_init_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [skip_init_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [skip_init_SUITE,
+ '$proplist',
+ {skip,"Skipped in init_per_suite/1"},
+ []]}},
+ {?eh,tc_done,{skip_init_SUITE,init_per_suite,
+ {skipped,"Skipped in init_per_suite/1"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_init_SUITE,init_per_suite,
+ {tc_user_skip,"Skipped in init_per_suite/1"},
+ []]}},
+ {?eh,tc_user_skip,{skip_init_SUITE,test_case,"Skipped in init_per_suite/1"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_init_SUITE,test_case,
+ {tc_user_skip,"Skipped in init_per_suite/1"},
+ []]}},
+ {?eh,test_stats,{0,0,{2,0}}},
+ {?eh,tc_user_skip,{skip_init_SUITE,end_per_suite,
+ "Skipped in init_per_suite/1"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_init_SUITE,end_per_suite,
+ {tc_user_skip,"Skipped in init_per_suite/1"},
+ []]}},
+
+ %% skip_req_SUITE is auto-skipped since a 'require' statement
+ %% returned by suite/0 is not fulfilled.
+ %% No group- or testcase-functions shall be called.
+ {?eh,tc_start,{skip_req_SUITE,init_per_suite}},
+ {?eh,tc_done,{skip_req_SUITE,init_per_suite,
+ {auto_skipped,{require_failed_in_suite0,
+ {not_available,whatever}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_req_SUITE,init_per_suite,
+ {tc_auto_skip,{require_failed_in_suite0,
+ {not_available,whatever}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_req_SUITE,test_case,{require_failed_in_suite0,
+ {not_available,whatever}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_req_SUITE,test_case,
+ {tc_auto_skip,{require_failed_in_suite0,
+ {not_available,whatever}}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,1}}},
+ {?eh,tc_auto_skip,{skip_req_SUITE,end_per_suite,
+ {require_failed_in_suite0,
+ {not_available,whatever}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_req_SUITE,end_per_suite,
+ {tc_auto_skip,{require_failed_in_suite0,
+ {not_available,whatever}}},
+ []]}},
+
+ %% skip_fail_SUITE is auto-skipped since the suite/0 function
+ %% retuns a faluty format.
+ %% No group- or testcase-functions shall be called.
+ {?eh,tc_start,{skip_fail_SUITE,init_per_suite}},
+ {?eh,tc_done,{skip_fail_SUITE,init_per_suite,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_fail_SUITE,init_per_suite,
+ {tc_auto_skip,
+ {failed,{error,{suite0_failed,bad_return_value}}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_fail_SUITE,test_case,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_fail_SUITE,test_case,
+ {tc_auto_skip,
+ {failed,{error,{suite0_failed,bad_return_value}}}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,2}}},
+ {?eh,tc_auto_skip,{skip_fail_SUITE,end_per_suite,
+ {failed,{error,{suite0_failed,bad_return_value}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_fail_SUITE,end_per_suite,
+ {tc_auto_skip,
+ {failed,{error,{suite0_failed,bad_return_value}}}},
+ []]}},
+
+ %% skip_group_SUITE
+ {?eh,tc_start,{skip_group_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ '_',
+ []]}},
+ {?eh,tc_done,{skip_group_SUITE,init_per_suite,ok}},
+
+ %% test_group_1 - auto_skip due to require failed
+ [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_1,[]}}},
+ {?eh,tc_done,
+ {skip_group_SUITE,{init_per_group,test_group_1,[]},
+ {auto_skipped,{require_failed,{not_available,whatever}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {init_per_group,test_group_1},
+ {tc_auto_skip,{require_failed,{not_available,whatever}}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_1},
+ {require_failed,{not_available,whatever}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {test_case,test_group_1},
+ {tc_auto_skip,{require_failed,{not_available,whatever}}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,3}}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_1},
+ {require_failed,{not_available,whatever}}}}],
+ %% The following appears to be outside of the group, but
+ %% that's only an implementation detail in
+ %% ct_test_support.erl - it does not know about events from
+ %% test suite specific hooks and regards the group ended with
+ %% the above tc_auto_skip-event for end_per_group.
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {end_per_group,test_group_1},
+ {tc_auto_skip,{require_failed,{not_available,whatever}}},
+ []]}},
+
+ %% test_group_2 - auto_skip due to failed return from group/1
+ [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_2,[]}}},
+ {?eh,tc_done,
+ {skip_group_SUITE,{init_per_group,test_group_2,[]},
+ {auto_skipped,{group0_failed,bad_return_value}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {init_per_group,test_group_2},
+ {tc_auto_skip,{group0_failed,bad_return_value}},
+ []]}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_2},
+ {group0_failed,bad_return_value}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {test_case,test_group_2},
+ {tc_auto_skip,{group0_failed,bad_return_value}},
+ []]}},
+ {?eh,test_stats,{0,0,{2,4}}},
+ {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_2},
+ {group0_failed,bad_return_value}}}],
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {end_per_group,test_group_2},
+ {tc_auto_skip,{group0_failed,bad_return_value}},
+ []]}},
+ %% test_group_3 - user_skip in init_per_group/2
+ [{?eh,tc_start,
+ {skip_group_SUITE,{init_per_group,test_group_3,[]}}},
+ {?eh,cth,{empty_cth,pre_init_per_group,
+ [skip_group_SUITE,test_group_3,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_group,
+ [skip_group_SUITE,test_group_3,'$proplist',
+ {skip,"Skipped in init_per_group/2"},
+ []]}},
+ {?eh,tc_done,{skip_group_SUITE,
+ {init_per_group,test_group_3,[]},
+ {skipped,"Skipped in init_per_group/2"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {init_per_group,test_group_3},
+ {tc_user_skip,"Skipped in init_per_group/2"},
+ []]}},
+ {?eh,tc_user_skip,{skip_group_SUITE,
+ {test_case,test_group_3},
+ "Skipped in init_per_group/2"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {test_case,test_group_3},
+ {tc_user_skip,"Skipped in init_per_group/2"},
+ []]}},
+ {?eh,test_stats,{0,0,{3,4}}},
+ {?eh,tc_user_skip,{skip_group_SUITE,
+ {end_per_group,test_group_3},
+ "Skipped in init_per_group/2"}}],
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_group_SUITE,
+ {end_per_group,test_group_3},
+ {tc_user_skip,"Skipped in init_per_group/2"},
+ []]}},
+
+ {?eh,tc_start,{skip_group_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [skip_group_SUITE,
+ '$proplist',
+ ok,[]]}},
+ {?eh,tc_done,{skip_group_SUITE,end_per_suite,ok}},
+
+
+ %% skip_case_SUITE has 4 test cases which are all skipped in
+ %% different ways
+ {?eh,tc_start,{skip_case_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ '_',
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,init_per_suite,ok}},
+
+ %% Skip in spec -> only on_tc_skip shall be called
+ {?eh,tc_user_skip,{skip_case_SUITE,skip_in_spec,"Skipped in spec"}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,skip_in_spec,
+ {tc_user_skip,"Skipped in spec"},
+ []]}},
+ {?eh,test_stats,{0,0,{4,4}}},
+
+ %% Skip in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,skip_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,skip_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,skip_in_init,
+ '$proplist',
+ {skip,"Skipped in init_per_testcase/2"},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,skip_in_init,
+ {skipped,"Skipped in init_per_testcase/2"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,skip_in_init,
+ {tc_user_skip,"Skipped in init_per_testcase/2"},
+ []]}},
+ {?eh,test_stats,{0,0,{5,4}}},
+
+ %% Fail in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,fail_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,fail_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,fail_in_init,
+ '$proplist',
+ {skip,{failed,'_'}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,fail_in_init,
+ {auto_skipped,{failed,'_'}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,fail_in_init,
+ {tc_auto_skip,{failed,'_'}},
+ []]}},
+ {?eh,test_stats,{0,0,{5,5}}},
+
+ %% Exit in init_per_testcase -> pre/post_end_per_testcase
+ %% shall not be called
+ {?eh,tc_start,{skip_case_SUITE,exit_in_init}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,exit_in_init,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,exit_in_init,
+ '$proplist',
+ {skip,{failed,'_'}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,exit_in_init,
+ {auto_skipped,{failed,'_'}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,exit_in_init,
+ {tc_auto_skip,{failed,'_'}},
+ []]}},
+ {?eh,test_stats,{0,0,{5,6}}},
+
+ %% Fail in end_per_testcase -> all hooks shall be called and
+ %% test shall succeed.
+ {?eh,tc_start,{skip_case_SUITE,fail_in_end}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ ok,
+ []]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,fail_in_end,
+ '$proplist',
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',
+ {test_case_failed,"Failed in end_per_testcase/2"}}}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,fail_in_end,
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',
+ {test_case_failed,"Failed in end_per_testcase/2"}}}}}},
+ {?eh,test_stats,{1,0,{5,6}}},
+
+ %% Exit in end_per_testcase -> all hooks shall be called and
+ %% test shall succeed.
+ {?eh,tc_start,{skip_case_SUITE,exit_in_end}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ ok,
+ []]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,exit_in_end,
+ '$proplist',
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',"Exit in end_per_testcase/2"}}},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,exit_in_end,
+ {failed,
+ {skip_case_SUITE,end_per_testcase,
+ {'EXIT',"Exit in end_per_testcase/2"}}}}},
+ {?eh,test_stats,{2,0,{5,6}}},
+
+ %% Skip in testcase function -> all callbacks shall be called
+ {?eh,tc_start,{skip_case_SUITE,skip_in_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ ok,[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [skip_case_SUITE,skip_in_case,
+ '$proplist',
+ {skip,"Skipped in test case function"},
+ []]}},
+ {?eh,tc_done,{skip_case_SUITE,skip_in_case,
+ {skipped,"Skipped in test case function"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,skip_in_case,
+ {tc_user_skip,"Skipped in test case function"},
+ []]}},
+ {?eh,test_stats,{2,0,{6,6}}},
+
+ %% Auto skip due to failed 'require' -> only the on_tc_skip
+ %% callback shall be called
+ {?eh,tc_start,{skip_case_SUITE,req_auto_skip}},
+ {?eh,tc_done,{skip_case_SUITE,req_auto_skip,
+ {auto_skipped,{require_failed,{not_available,whatever}}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,req_auto_skip,
+ {tc_auto_skip,{require_failed,{not_available,whatever}}},
+ []]}},
+ {?eh,test_stats,{2,0,{6,7}}},
+
+ %% Auto skip due to failed testcase/0 function -> only the
+ %% on_tc_skip callback shall be called
+ {?eh,tc_start,{skip_case_SUITE,fail_auto_skip}},
+ {?eh,tc_done,{skip_case_SUITE,fail_auto_skip,
+ {auto_skipped,{testcase0_failed,bad_return_value}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [skip_case_SUITE,fail_auto_skip,
+ {tc_auto_skip,{testcase0_failed,bad_return_value}},
+ []]}},
+ {?eh,test_stats,{2,0,{6,8}}},
+
+ {?eh,tc_start,{skip_case_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ []]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [skip_case_SUITE,
+ '$proplist',
+ ok,[]]}},
+ {?eh,tc_done,{skip_case_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
+test_events(failed_sequence) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events =
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[seq_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [seq_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_start,{seq_SUITE,test_case_1}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [seq_SUITE,test_case_1,'$proplist',
+ {error,failed_on_purpose},[]]}},
+ {?eh,tc_done,{seq_SUITE,test_case_1,{failed,{error,failed_on_purpose}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,
+ [seq_SUITE,test_case_1,failed_on_purpose,[]]}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{seq_SUITE,test_case_2}},
+ {?eh,tc_done,{seq_SUITE,test_case_2,
+ {auto_skipped,{sequence_failed,seq1,test_case_1}}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [seq_SUITE,test_case_2,
+ {tc_auto_skip,{sequence_failed,seq1,test_case_1}},
+ []]}},
+ {?eh,test_stats,{0,1,{0,1}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[seq_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,[seq_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
+test_events(repeat_force_stop) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events=
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[repeat_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [repeat_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_start,{repeat_SUITE,test_case_1}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,cth,{empty_cth,pre_end_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_testcase,
+ [repeat_SUITE,test_case_1,'$proplist',ok,[]]}},
+ {?eh,tc_done,{repeat_SUITE,test_case_1,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ {?eh,tc_start,{repeat_SUITE,test_case_2}},
+ {?eh,tc_done,{repeat_SUITE,test_case_2,
+ {auto_skipped,
+ "Repeated test stopped by force_stop option"}}},
+ {?eh,cth,{empty_cth,on_tc_skip,
+ [repeat_SUITE,test_case_2,
+ {tc_auto_skip,"Repeated test stopped by force_stop option"},
+ []]}},
+ {?eh,test_stats,{1,0,{0,1}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[repeat_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [repeat_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
+test_events(config_clash) ->
+ %% skip_cth.erl will send a 'cth_error' event if a hook is
+ %% erroneously called. Therefore, all Events are changed to
+ %% {negative,{?eh,cth_error,'_'},Event}
+ %% at the end of this function.
+ Events =
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,tc_start,{ct_framework,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,
+ [config_clash_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [config_clash_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+ {?eh,tc_start,{config_clash_SUITE,test_case_1}},
+ {?eh,tc_done,{config_clash_SUITE,test_case_1,
+ {failed,{error,{config_name_already_in_use,[aa]}}}}},
+ {?eh,cth,{empty_cth,on_tc_fail,
+ [config_clash_SUITE,test_case_1,
+ {config_name_already_in_use,[aa]},
+ []]}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{ct_framework,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,
+ [config_clash_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [config_clash_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ],
+ %% Make sure no 'cth_error' events are received!
+ [{negative,{?eh,cth_error,'_'},E} || E <- Events];
+
test_events(ok) ->
ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl
new file mode 100644
index 0000000000..5b50548694
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl
@@ -0,0 +1,62 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(all_hook_callbacks_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_group(Config) ->
+ Config.
+
+end_per_group(_Config) ->
+ ok.
+
+init_per_testcase(skip_case, Config) ->
+ {skip,"Skipped in init_per_testcase/2"};
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [{group,test_group},test_case,skip_case].
+
+groups() ->
+ [{test_group,[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
+
+skip_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl
new file mode 100644
index 0000000000..f74c757cc1
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl
@@ -0,0 +1,43 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(config_clash_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [{require,aa,yy},{default_config,yy,"this is a default value"}].
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case_1].
+
+%% Test cases starts here.
+test_case_1() ->
+ [{require,aa,xx}].
+test_case_1(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl
new file mode 100644
index 0000000000..7cdaf2024b
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl
@@ -0,0 +1,51 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_no_end_config_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+%%% This suite is used to verify that all pre/post_end_per_* callbacks
+%%% are called with correct SuiteName even if no end_per_* config
+%%% function exist in the suite, and that the non-exported config
+%%% functions fail with 'undef'.
+
+init_per_suite(Config) ->
+ Config.
+
+init_per_group(_Group,Config) ->
+ Config.
+
+init_per_testcase(_TC,Config) ->
+ Config.
+
+all() ->
+ [test_case_1, {group,test_group}].
+
+groups() ->
+ [{test_group,[],[test_case_2]}].
+
+test_case_1(Config) ->
+ ok.
+
+test_case_2(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl
new file mode 100644
index 0000000000..43c062d66f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl
@@ -0,0 +1,54 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_no_init_config_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+%%% This suite is used to verify that all
+%%% pre/post_init_per_group/testcase callbacks are called with correct
+%%% SuiteName even if no init_per_group/testcase function exist in the
+%%% suite, and that the non-exported config functions fail with 'undef'.
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+end_per_group(_Group,Config) ->
+ Config.
+
+end_per_testcase(_TC,Config) ->
+ Config.
+
+all() ->
+ [test_case_1, {group,test_group}].
+
+groups() ->
+ [{test_group,[],[test_case_2]}].
+
+test_case_1(Config) ->
+ ok.
+
+test_case_2(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl
new file mode 100644
index 0000000000..85dfe8ca4b
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl
@@ -0,0 +1,39 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_no_init_suite_config_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+%%% This suite is used to verify that pre/post_init_per_suite
+%%% callbacks are called with correct SuiteName even if no
+%%% init_per_suite function exist in the suite, and that the
+%%% non-exported config function fails with 'undef'.
+
+end_per_suite(Config) ->
+ Config.
+
+all() ->
+ [test_case].
+
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl
index c00eb5cf93..37742f0d20 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl
@@ -44,18 +44,18 @@
-export([pre_end_per_suite/3]).
-export([post_end_per_suite/4]).
--export([pre_init_per_group/3]).
--export([post_init_per_group/4]).
--export([pre_end_per_group/3]).
--export([post_end_per_group/4]).
+-export([pre_init_per_group/4]).
+-export([post_init_per_group/5]).
+-export([pre_end_per_group/4]).
+-export([post_end_per_group/5]).
--export([pre_init_per_testcase/3]).
--export([post_init_per_testcase/4]).
--export([pre_end_per_testcase/3]).
--export([post_end_per_testcase/4]).
+-export([pre_init_per_testcase/4]).
+-export([post_init_per_testcase/5]).
+-export([pre_end_per_testcase/4]).
+-export([post_end_per_testcase/5]).
--export([on_tc_fail/3]).
--export([on_tc_skip/3]).
+-export([on_tc_fail/4]).
+-export([on_tc_skip/4]).
-export([terminate/1]).
@@ -154,150 +154,160 @@ post_end_per_suite(Suite,Config,Return,State) ->
%% @doc Called before each init_per_group.
%% You can change the config in this function.
--spec pre_init_per_group(Group :: atom(),
- Config :: config(),
- State :: #state{}) ->
+-spec pre_init_per_group(Suite :: atom(),
+ Group :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
{config() | skip_or_fail(), NewState :: #state{}}.
-pre_init_per_group(Group,Config,State) ->
+pre_init_per_group(Suite,Group,Config,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, pre_init_per_group,
- [Group,Config,State]}}),
- ct:log("~w:pre_init_per_group(~w) called", [?MODULE,Group]),
+ [Suite,Group,Config,State]}}),
+ ct:log("~w:pre_init_per_group(~w,~w) called", [?MODULE,Suite,Group]),
{Config, State}.
%% @doc Called after each init_per_group.
%% You can change the return value in this function.
--spec post_init_per_group(Group :: atom(),
+-spec post_init_per_group(Suite :: atom(),
+ Group :: atom(),
Config :: config(),
Return :: config() | skip_or_fail(),
State :: #state{}) ->
{config() | skip_or_fail(), NewState :: #state{}}.
-post_init_per_group(Group,Config,Return,State) ->
+post_init_per_group(Suite,Group,Config,Return,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, post_init_per_group,
- [Group,Config,Return,State]}}),
- ct:log("~w:post_init_per_group(~w) called", [?MODULE,Group]),
+ [Suite,Group,Config,Return,State]}}),
+ ct:log("~w:post_init_per_group(~w,~w) called", [?MODULE,Suite,Group]),
{Return, State}.
%% @doc Called after each end_per_group. The config/state can be changed here,
%% though it will only affect the *end_per_group functions.
--spec pre_end_per_group(Group :: atom(),
+-spec pre_end_per_group(Suite :: atom(),
+ Group :: atom(),
Config :: config() | skip_or_fail(),
State :: #state{}) ->
{ok | skip_or_fail(), NewState :: #state{}}.
-pre_end_per_group(Group,Config,State) ->
+pre_end_per_group(Suite,Group,Config,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, pre_end_per_group,
- [Group,Config,State]}}),
- ct:log("~w:pre_end_per_group(~w) called", [?MODULE,Group]),
+ [Suite,Group,Config,State]}}),
+ ct:log("~w:pre_end_per_group(~w~w) called", [?MODULE,Suite,Group]),
{Config, State}.
%% @doc Called after each end_per_group. Note that the config cannot be
%% changed here, only the status of the group.
--spec post_end_per_group(Group :: atom(),
+-spec post_end_per_group(Suite :: atom(),
+ Group :: atom(),
Config :: config(),
Return :: term(),
State :: #state{}) ->
{ok | skip_or_fail(), NewState :: #state{}}.
-post_end_per_group(Group,Config,Return,State) ->
+post_end_per_group(Suite,Group,Config,Return,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, post_end_per_group,
- [Group,Config,Return,State]}}),
- ct:log("~w:post_end_per_group(~w) called", [?MODULE,Group]),
+ [Suite,Group,Config,Return,State]}}),
+ ct:log("~w:post_end_per_group(~w,~w) called", [?MODULE,Suite,Group]),
{Return, State}.
%% @doc Called before init_per_testcase/2 for each test case.
%% You can change the config in this function.
--spec pre_init_per_testcase(TC :: atom(),
- Config :: config(),
- State :: #state{}) ->
+-spec pre_init_per_testcase(Suite :: atom(),
+ TC :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
{config() | skip_or_fail(), NewState :: #state{}}.
-pre_init_per_testcase(TC,Config,State) ->
+pre_init_per_testcase(Suite,TC,Config,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, pre_init_per_testcase,
- [TC,Config,State]}}),
- ct:log("~w:pre_init_per_testcase(~w) called", [?MODULE,TC]),
+ [Suite,TC,Config,State]}}),
+ ct:log("~w:pre_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),
{Config, State}.
%% @doc Called after init_per_testcase/2, and before the test case.
--spec post_init_per_testcase(TC :: atom(),
+-spec post_init_per_testcase(Suite :: atom(),
+ TC :: atom(),
Config :: config(),
Return :: config() | skip_or_fail(),
State :: #state{}) ->
{config() | skip_or_fail(), NewState :: #state{}}.
-post_init_per_testcase(TC,Config,Return,State) ->
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, post_init_per_testcase,
- [TC,Config,Return,State]}}),
- ct:log("~w:post_init_per_testcase(~w) called", [?MODULE,TC]),
+ [Suite,TC,Config,Return,State]}}),
+ ct:log("~w:post_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),
{Return, State}.
%% @doc Called before end_per_testacse/2. No skip or fail allowed here,
%% only config additions.
--spec pre_end_per_testcase(TC :: atom(),
- Config :: config(),
- State :: #state{}) ->
+-spec pre_end_per_testcase(Suite :: atom(),
+ TC :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
{config(), NewState :: #state{}}.
-pre_end_per_testcase(TC,Config,State) ->
+pre_end_per_testcase(Suite,TC,Config,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, pre_end_per_testcase,
- [TC,Config,State]}}),
- ct:log("~w:pre_end_per_testcase(~w) called", [?MODULE,TC]),
+ [Suite,TC,Config,State]}}),
+ ct:log("~w:pre_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),
{Config, State}.
%% @doc Called after end_per_testcase/2 for each test case. Note that
%% the config cannot be changed here, only the status of the test case.
--spec post_end_per_testcase(TC :: atom(),
+-spec post_end_per_testcase(Suite :: atom(),
+ TC :: atom(),
Config :: config(),
Return :: term(),
State :: #state{}) ->
{ok | skip_or_fail(), NewState :: #state{}}.
-post_end_per_testcase(TC,Config,Return,State) ->
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, post_end_per_testcase,
- [TC,Config,Return,State]}}),
- ct:log("~w:post_end_per_testcase(~w) called", [?MODULE,TC]),
+ [Suite,TC,Config,Return,State]}}),
+ ct:log("~w:post_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),
{Return, State}.
%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
%% post_end_per_group and post_end_per_tc if the suite, group or test case failed.
%% This function should be used for extra cleanup which might be needed.
%% It is not possible to modify the config or the status of the test run.
--spec on_tc_fail(TC :: init_per_suite | end_per_suite |
+-spec on_tc_fail(Suite :: atom(),
+ TC :: init_per_suite | end_per_suite |
init_per_group | end_per_group | atom() |
{Function :: atom(), GroupName :: atom()},
Reason :: term(), State :: #state{}) -> NewState :: #state{}.
-on_tc_fail(TC, Reason, State) ->
+on_tc_fail(Suite, TC, Reason, State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, on_tc_fail,
- [TC,Reason,State]}}),
- ct:log("~w:on_tc_fail(~w) called", [?MODULE,TC]),
+ [Suite,TC,Reason,State]}}),
+ ct:log("~w:on_tc_fail(~w,~w) called", [?MODULE,Suite,TC]),
State.
%% @doc Called when a test case is skipped by either user action
%% or due to an init function failing. Test case can be
%% end_per_suite, init_per_group, end_per_group and the actual test cases.
--spec on_tc_skip(TC :: end_per_suite |
+-spec on_tc_skip(Suite :: atom(),
+ TC :: end_per_suite |
init_per_group | end_per_group | atom() |
{Function :: atom(), GroupName :: atom()},
{tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |
{tc_user_skip, {skipped, Reason :: term()}},
State :: #state{}) -> NewState :: #state{}.
-on_tc_skip(TC, Reason, State) ->
+on_tc_skip(Suite, TC, Reason, State) ->
gen_event:notify(
?CT_EVMGR_REF, #event{ name = cth, node = node(),
data = {?MODULE, on_tc_skip,
- [TC,Reason,State]}}),
- ct:log("~w:on_tc_skip(~w) called", [?MODULE,TC]),
+ [Suite,TC,Reason,State]}}),
+ ct:log("~w:on_tc_skip(~w,~w) called", [?MODULE,Suite,TC]),
State.
%% @doc Called when the scope of the CTH is done, this depends on
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl
index 559b22bc9f..141b933697 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl
@@ -45,29 +45,29 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl
index 51202443bf..07d7c84ed5 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl
@@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl
new file mode 100644
index 0000000000..59a3d5cbf9
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl
@@ -0,0 +1,81 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fallback_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(fallback_nosuite,Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(fallback_nosuite,Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(fallback_nosuite,Group,Config,State).
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(fallback_nosuite,Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(fallback_nosuite,TC,Config,State).
+
+post_init_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(fallback_nosuite,TC,Config,Return,State).
+
+pre_end_per_testcase(TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(fallback_nosuite,TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(fallback_nosuite,TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(fallback_nosuite,TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(fallback_nosuite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl
index b49cbe7fb4..679f076f3a 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl
@@ -29,13 +29,13 @@
%% CT Hooks
-export([init/2]).
-export([terminate/1]).
--export([on_tc_skip/3]).
+-export([on_tc_skip/4]).
init(Id, Opts) ->
empty_cth:init(Id, Opts).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite, TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl
index a687743641..95bb76b4c1 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl
@@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl
index 4d9c60f1ca..3562d39967 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl
@@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl
new file mode 100644
index 0000000000..fded4c02ab
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl
@@ -0,0 +1,42 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(repeat_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case_1, test_case_2].
+
+%% Test cases starts here.
+test_case_1(_Config) ->
+ timer:sleep(10000),
+ ok.
+
+test_case_2(_Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl
index 494f398fc1..b9d9d4cec1 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl
@@ -48,35 +48,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl
new file mode 100644
index 0000000000..6d1302fd35
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl
@@ -0,0 +1,45 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(seq_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [{sequence,seq1}].
+
+sequences() ->
+ [{seq1,[test_case_1,test_case_2]}].
+
+%% Test cases starts here.
+test_case_1(_Config) ->
+ exit(failed_on_purpose).
+
+test_case_2(_Config) ->
+ ct:fail("This test shall never be run since test_case_1 fails "
+ "and they are run in sequence").
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec
new file mode 100644
index 0000000000..a271c5e8b2
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec
@@ -0,0 +1,8 @@
+{suites,".",[all_hook_callbacks_SUITE,
+ skip_init_SUITE,
+ skip_req_SUITE,
+ skip_fail_SUITE,
+ skip_group_SUITE,
+ skip_case_SUITE]}.
+{skip_suites,".",all_hook_callbacks_SUITE,"Skipped in spec"}.
+{skip_cases,".",skip_case_SUITE,skip_in_spec,"Skipped in spec"}.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
new file mode 100644
index 0000000000..dad80ae914
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl
@@ -0,0 +1,106 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_case_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(skip_in_init,Config) ->
+ {skip,"Skipped in init_per_testcase/2"};
+init_per_testcase(fail_in_init,Config) ->
+ ct:fail("Failed in init_per_testcase/2");
+init_per_testcase(exit_in_init,Config) ->
+ exit(self(),"Exit in init_per_testcase/2");
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(fail_in_end,_) ->
+ ct:fail("Failed in end_per_testcase/2");
+end_per_testcase(exit_in_end,_) ->
+ exit(self(),"Exit in end_per_testcase/2");
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [skip_in_spec,
+ skip_in_init,
+ fail_in_init,
+ exit_in_init,
+ fail_in_end,
+ exit_in_end,
+ skip_in_case,
+ req_auto_skip,
+ fail_auto_skip
+ ].
+
+%% Test cases starts here.
+skip_in_spec(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall be skipped in the test spec.").
+
+skip_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall be skipped in init_per_testcase/2.").
+
+fail_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall fail in init_per_testcase/2.").
+
+exit_in_init(Config) ->
+ ct:fail("This test shall never be run. "
+ "It shall exit in init_per_testcase/2.").
+
+fail_in_end(Config) ->
+ ok.
+
+exit_in_end(Config) ->
+ ok.
+
+skip_in_case(Config) ->
+ {skip,"Skipped in test case function"}.
+
+req_auto_skip() ->
+ [{require,whatever}].
+req_auto_skip(Config) ->
+ ct:fail("This test shall never be run due to "
+ "failed require").
+
+fail_auto_skip() ->
+ faulty_return_value.
+fail_auto_skip(Config) ->
+ ct:fail("This test shall never be run due to "
+ "faulty return from info function").
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
new file mode 100644
index 0000000000..16f015fe7a
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl
@@ -0,0 +1,182 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+%% Send a cth_error event if a callback is called with unexpected arguments
+-define(fail(Info),
+ gen_event:notify(
+ ?CT_EVMGR_REF,
+ #event{ name = cth_error,
+ node = node(),
+ data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})).
+
+%% CT Hooks
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ Suite==skip_init_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==skip_case_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ Suite==skip_init_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==skip_case_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ Suite==skip_case_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ Suite==skip_case_SUITE
+ orelse Suite==skip_group_SUITE
+ orelse Suite==seq_SUITE
+ orelse Suite==repeat_SUITE
+ orelse Suite==config_clash_SUITE
+ orelse ?fail(Suite),
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ (Suite==skip_group_SUITE andalso Group==test_group_3)
+ orelse ?fail({Suite,Group}),
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ (Suite==skip_group_SUITE andalso Group==test_group_3)
+ orelse ?fail({Suite,Group}),
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ ?fail({Suite,Group}),
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ ?fail({Suite,Group}),
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ (Suite==skip_case_SUITE andalso (TC==skip_in_init
+ orelse TC==fail_in_init
+ orelse TC==exit_in_init
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end
+ orelse TC==skip_in_case))
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ (Suite==skip_case_SUITE andalso (TC==skip_in_init
+ orelse TC==fail_in_init
+ orelse TC==exit_in_init
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end
+ orelse TC==skip_in_case))
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ (Suite==skip_case_SUITE andalso (TC==skip_in_case
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end))
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ (Suite==skip_case_SUITE andalso (TC==skip_in_case
+ orelse TC==fail_in_end
+ orelse TC==exit_in_end))
+ orelse (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC,Reason,State) ->
+ (Suite==seq_SUITE andalso TC==test_case_1)
+ orelse (Suite==config_clash_SUITE andalso TC==test_case_1)
+ orelse ?fail({Suite,TC}),
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(all_hook_callbacks_SUITE=Suite,all=TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(Suite,TC,Reason,State)
+ when (Suite==skip_init_SUITE
+ orelse Suite==skip_req_SUITE
+ orelse Suite==skip_fail_SUITE)
+ andalso
+ (TC==init_per_suite
+ orelse TC==test_case
+ orelse TC==end_per_suite) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(skip_group_SUITE=Suite,TC={C,G},Reason,State)
+ when (C==init_per_group orelse C==test_case orelse C==end_per_group) andalso
+ (G==test_group_1 orelse G==test_group_2 orelse G==test_group_3) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State)
+ when TC==skip_in_spec;
+ TC==skip_in_init;
+ TC==fail_in_init;
+ TC==exit_in_init;
+ TC==skip_in_case;
+ TC==req_auto_skip;
+ TC==fail_auto_skip ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(Suite,TC,Reason,State)
+ when (Suite==seq_SUITE andalso TC==test_case_2)
+ orelse (Suite==repeat_SUITE andalso TC==test_case_2) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State);
+on_tc_skip(Suite,TC,Reason,State) ->
+ ?fail({Suite,TC}),
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl
new file mode 100644
index 0000000000..9f5dfee6b9
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl
@@ -0,0 +1,53 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_fail_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ faulty_return_value.
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl
new file mode 100644
index 0000000000..d3b848bfbd
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl
@@ -0,0 +1,64 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_group_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+group(test_group_1) ->
+ [{require,whatever}];
+group(test_group_2) ->
+ faulty_return_value;
+group(_) ->
+ [].
+
+init_per_group(test_group_3,Config) ->
+ {skip,"Skipped in init_per_group/2"};
+init_per_group(_,Config) ->
+ ct:fail("This shall never be run due to auto_skip from group/1").
+
+end_per_group(_,_) ->
+ ct:fail("This shall never be run").
+
+all() ->
+ [{group,test_group_1},
+ {group,test_group_2},
+ {group,test_group_3}].
+
+groups() ->
+ [{test_group_1,[test_case]},
+ {test_group_2,[test_case]},
+ {test_group_3,[test_case]}].
+
+%% Test cases starts here.
+test_case(_Config) ->
+ ct:fail("This test case shall never be run due to skip on group level").
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl
new file mode 100644
index 0000000000..70305421ac
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl
@@ -0,0 +1,53 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_init_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [].
+
+init_per_suite(Config) ->
+ {skip,"Skipped in init_per_suite/1"}.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl
index d5b347e723..48a2d70e22 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl
@@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl
index 36abac0bf8..d638954d3c 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl
@@ -46,36 +46,36 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State),
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State),
{{skip, "Test skip"}, State}.
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl
new file mode 100644
index 0000000000..e1d261d59a
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl
@@ -0,0 +1,79 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_pre_init_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State),
+ {{skip, "Skipped in pre_init_per_testcase"}, State}.
+
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl
index fa510b2d54..d7b07ee33c 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl
@@ -46,35 +46,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl
new file mode 100644
index 0000000000..bc69dd5ea4
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl
@@ -0,0 +1,53 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(skip_req_SUITE).
+
+-compile(export_all).
+
+-include("ct.hrl").
+
+suite() ->
+ [{require,whatever}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ ok.
+
+init_per_group(_,Config) ->
+ Config.
+
+end_per_group(_,_) ->
+ ok.
+
+init_per_testcase(_,Config) ->
+ Config.
+
+end_per_testcase(_,_) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl
index 7ec0d458b6..c6e0419c50 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl
@@ -48,44 +48,44 @@ post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State),
{Return, [post_end_per_suite|State]}.
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State),
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State),
{Config, [pre_init_per_group|State]}.
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State),
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State),
{Return, [post_init_per_group|State]}.
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State),
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State),
{Config, [pre_end_per_group|State]}.
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State),
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State),
{Return, [post_end_per_group|State]}.
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State),
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State),
{Config, [pre_init_per_testcase|State]}.
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State),
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State),
{Return, [post_init_per_testcase|State]}.
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State),
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State),
{Config, [pre_end_per_testcase|State]}.
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State),
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),
{Return, [post_end_per_testcase|State]}.
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State),
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State),
[on_tc_fail|State].
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State),
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State),
[on_tc_skip|State].
terminate(State) ->
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl
index 2b9e726819..10a7047899 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl
@@ -44,35 +44,35 @@ pre_end_per_suite(Suite,Config,State) ->
post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl
index d48981f667..f933c7702e 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl
@@ -50,43 +50,43 @@ post_end_per_suite(Suite,Config,Return,State) ->
NewConfig = [{post_end_per_suite,?now}|Config],
{NewConfig,NewConfig}.
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State),
+pre_init_per_group(Suite, Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State),
{[{pre_init_per_group,?now}|Config],State}.
-post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State),
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State),
{[{post_init_per_group,?now}|Return],State}.
-pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State),
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State),
{[{pre_end_per_group,?now}|Config],State}.
-post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State),
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State),
{[{post_end_per_group,?now}|Config],State}.
-pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State),
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State),
{[{pre_init_per_testcase,?now}|Config],State}.
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State),
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State),
{[{post_init_per_testcase,?now}|Config],State}.
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State),
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State),
{[{pre_end_per_testcase,?now}|Config],State}.
-post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State),
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),
{[{post_end_per_testcase,?now}|Config],State}.
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl
index 71d84781e0..b29256a77e 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl
@@ -60,37 +60,37 @@ post_end_per_suite(Suite,Config,Return,State) ->
ct_no_config_SUITE = ct:get_config(suite_cfg),
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
+pre_init_per_group(Suite,Group,Config,State) ->
true = ?val(post_init_per_suite, Config),
ct_no_config_SUITE = ct:get_config(suite_cfg),
test_group = ct:get_config(group_cfg),
- empty_cth:pre_init_per_group(Group,
+ empty_cth:pre_init_per_group(Suite,Group,
[{pre_init_per_group,true} | Config],
State).
-post_init_per_group(Group,Config,Return,State) ->
+post_init_per_group(Suite,Group,Config,Return,State) ->
true = ?val(pre_init_per_group, Return),
test_group = ct:get_config(group_cfg),
- empty_cth:post_init_per_group(Group,
+ empty_cth:post_init_per_group(Suite,Group,
Config,
[{post_init_per_group,true} | Return],
State).
-pre_end_per_group(Group,Config,State) ->
+pre_end_per_group(Suite,Group,Config,State) ->
true = ?val(post_init_per_group, Config),
ct_no_config_SUITE = ct:get_config(suite_cfg),
test_group = ct:get_config(group_cfg),
- empty_cth:pre_end_per_group(Group,
+ empty_cth:pre_end_per_group(Suite,Group,
[{pre_end_per_group,true} | Config],
State).
-post_end_per_group(Group,Config,Return,State) ->
+post_end_per_group(Suite,Group,Config,Return,State) ->
true = ?val(pre_end_per_group, Config),
ct_no_config_SUITE = ct:get_config(suite_cfg),
test_group = ct:get_config(group_cfg),
- empty_cth:post_end_per_group(Group,Config,Return,State).
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
+pre_init_per_testcase(Suite,TC,Config,State) ->
true = ?val(post_init_per_suite, Config),
case ?val(name, ?val(tc_group_properties, Config)) of
undefined ->
@@ -102,19 +102,19 @@ pre_init_per_testcase(TC,Config,State) ->
ct_no_config_SUITE = ct:get_config(suite_cfg),
CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"),
TC = ct:get_config(CfgKey),
- empty_cth:pre_init_per_testcase(TC,
+ empty_cth:pre_init_per_testcase(Suite,TC,
[{pre_init_per_testcase,true} | Config],
State).
%%! TODO: Verify Config also in post_init and pre_end!
-post_init_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
- empty_cth:pre_end_per_testcase(TC,Config,State).
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
true = ?val(post_init_per_suite, Config),
true = ?val(pre_init_per_testcase, Config),
case ?val(name, ?val(tc_group_properties, Config)) of
@@ -127,13 +127,13 @@ post_end_per_testcase(TC,Config,Return,State) ->
ct_no_config_SUITE = ct:get_config(suite_cfg),
CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"),
TC = ct:get_config(CfgKey),
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl
index 9abd2e5e83..42e086b96e 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl
@@ -62,43 +62,43 @@ post_end_per_suite(Suite,Config,Return,State) ->
check_dirs(State,Config),
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
+pre_init_per_group(Suite,Group,Config,State) ->
check_dirs(State,Config),
- empty_cth:pre_init_per_group(Group,Config,State).
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
-post_init_per_group(Group,Config,Return,State) ->
+post_init_per_group(Suite,Group,Config,Return,State) ->
check_dirs(State,Return),
- empty_cth:post_init_per_group(Group,Config,Return,State).
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
-pre_end_per_group(Group,Config,State) ->
+pre_end_per_group(Suite,Group,Config,State) ->
check_dirs(State,Config),
- empty_cth:pre_end_per_group(Group,Config,State).
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
-post_end_per_group(Group,Config,Return,State) ->
+post_end_per_group(Suite,Group,Config,Return,State) ->
check_dirs(State,Config),
- empty_cth:post_end_per_group(Group,Config,Return,State).
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
-pre_init_per_testcase(TC,Config,State) ->
+pre_init_per_testcase(Suite,TC,Config,State) ->
check_dirs(State,Config),
- empty_cth:pre_init_per_testcase(TC,Config,State).
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
-post_init_per_testcase(TC,Config,Return,State) ->
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
check_dirs(State,Config),
- empty_cth:post_init_per_testcase(TC,Config,Return,State).
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
-pre_end_per_testcase(TC,Config,State) ->
+pre_end_per_testcase(Suite,TC,Config,State) ->
check_dirs(State,Config),
- empty_cth:pre_end_per_testcase(TC,Config,State).
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
-post_end_per_testcase(TC,Config,Return,State) ->
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
check_dirs(State,Config),
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
-on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
-on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
index f8b6a379f6..76611a2db3 100644
--- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl
+++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl
@@ -363,14 +363,17 @@ skip_first_tc1(Suite) ->
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,1}}},
+ {?eh,tc_start,{Suite,{init_per_group,g,[]}}},
{?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},
{?eh,tc_auto_skip,{Suite,{tc1,g},?skip_reason}},
{?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_auto_skip,{Suite,{tc2,g},?skip_reason}},
{?eh,test_stats,{'_',0,{0,3}}},
{?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}},
+ {?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,4}}},
{?eh,tc_start,{Suite,end_per_suite}},
@@ -390,10 +393,12 @@ skip_tc1_in_group(Suite) ->
{?eh,tc_start,{Suite,tc1}},
{?eh,tc_done,{Suite,tc1,ok}},
{?eh,test_stats,{'_',0,{0,0}}},
+ {?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,1}}},
{?eh,tc_start,{Suite,{end_per_group,g,[]}}},
{?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}],
+ {?eh,tc_start,{Suite,tc2}},
{?eh,tc_done,{Suite,tc2,?skipped}},
{?eh,test_stats,{'_',0,{0,2}}},
{?eh,tc_start,{Suite,end_per_suite}},
diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl
index 42ec685c16..884217afc2 100644
--- a/lib/common_test/test/ct_surefire_SUITE.erl
+++ b/lib/common_test/test/ct_surefire_SUITE.erl
@@ -73,7 +73,9 @@ all() ->
relative_path,
url,
logdir,
- fail_pre_init_per_suite
+ fail_pre_init_per_suite,
+ skip_case_in_spec,
+ skip_suite_in_spec
].
%%--------------------------------------------------------------------
@@ -119,6 +121,18 @@ fail_pre_init_per_suite(Config) when is_list(Config) ->
run(fail_pre_init_per_suite,[fail_pre_init_per_suite,
{cth_surefire,[{path,Path}]}],Path,Config,[],Suites).
+skip_case_in_spec(Config) ->
+ DataDir = ?config(data_dir,Config),
+ Spec = filename:join(DataDir,"skip_one_case.spec"),
+ Path = "skip_case_in_spec.xml",
+ run_spec(skip_case_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec).
+
+skip_suite_in_spec(Config) ->
+ DataDir = ?config(data_dir,Config),
+ Spec = filename:join(DataDir,"skip_one_suite.spec"),
+ Path = "skip_suite_in_spec.xml",
+ run_spec(skip_suite_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec).
+
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
@@ -129,8 +143,15 @@ run(Case,CTHs,Report,Config,ExtraOpts) ->
Suite = filename:join(DataDir, "surefire_SUITE"),
run(Case,CTHs,Report,Config,ExtraOpts,Suite).
run(Case,CTHs,Report,Config,ExtraOpts,Suite) ->
- {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts],
- Config),
+ Test = [{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts],
+ do_run(Case, Report, Test, Config).
+
+run_spec(Case,CTHs,Report,Config,Spec) ->
+ Test = [{spec,Spec},{ct_hooks,CTHs},{label,Case}],
+ do_run(Case, Report, Test, Config).
+
+do_run(Case, Report, Test, Config) ->
+ {Opts,ERPid} = setup(Test, Config),
ok = execute(Case, Opts, ERPid, Config),
LogDir =
case lists:keyfind(logdir,1,Opts) of
@@ -201,7 +222,10 @@ test_suite_events(pass_SUITE) ->
{?eh,test_stats,{1,0,{0,0}}},
{?eh,tc_start,{ct_framework,end_per_suite}},
{?eh,tc_done,{ct_framework,end_per_suite,ok}}];
-test_suite_events(_) ->
+test_suite_events(skip_all_surefire_SUITE) ->
+ [{?eh,tc_user_skip,{skip_all_surefire_SUITE,all,"skipped in spec"}},
+ {?eh,test_stats,{0,0,{1,0}}}];
+test_suite_events(Test) ->
[{?eh,tc_start,{surefire_SUITE,init_per_suite}},
{?eh,tc_done,{surefire_SUITE,init_per_suite,ok}},
{?eh,tc_start,{surefire_SUITE,tc_ok}},
@@ -210,46 +234,55 @@ test_suite_events(_) ->
{?eh,tc_start,{surefire_SUITE,tc_fail}},
{?eh,tc_done,{surefire_SUITE,tc_fail,
{failed,{error,{test_case_failed,"this test should fail"}}}}},
- {?eh,test_stats,{1,1,{0,0}}},
- {?eh,tc_start,{surefire_SUITE,tc_skip}},
- {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}},
- {?eh,test_stats,{1,1,{1,0}}},
- {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
- {?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
- {auto_skipped,{require_failed,'_'}}}},
- {?eh,test_stats,{1,1,{1,1}}},
- [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}},
- {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}},
- {?eh,tc_start,{surefire_SUITE,tc_ok}},
- {?eh,tc_done,{surefire_SUITE,tc_ok,ok}},
- {?eh,test_stats,{2,1,{1,1}}},
- {?eh,tc_start,{surefire_SUITE,tc_fail}},
- {?eh,tc_done,{surefire_SUITE,tc_fail,
- {failed,{error,{test_case_failed,"this test should fail"}}}}},
- {?eh,test_stats,{2,2,{1,1}}},
- {?eh,tc_start,{surefire_SUITE,tc_skip}},
- {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}},
- {?eh,test_stats,{2,2,{2,1}}},
- {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
- {?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
- {auto_skipped,{require_failed,'_'}}}},
- {?eh,test_stats,{2,2,{2,2}}},
- {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}},
- {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}],
- [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}},
- {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]},
- {failed,{error,all_cases_should_be_skipped}}}},
- {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail},
- {failed,
- {surefire_SUITE,init_per_group,
- {'EXIT',all_cases_should_be_skipped}}}}},
- {?eh,test_stats,{2,2,{2,3}}},
- {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail},
- {failed,
- {surefire_SUITE,init_per_group,
- {'EXIT',all_cases_should_be_skipped}}}}}],
- {?eh,tc_start,{surefire_SUITE,end_per_suite}},
- {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}].
+ {?eh,test_stats,{1,1,{0,0}}}] ++
+ tc_skip_events(Test,undefined) ++
+ [{?eh,test_stats,{1,1,{1,0}}},
+ {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
+ {?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
+ {auto_skipped,{require_failed,'_'}}}},
+ {?eh,test_stats,{1,1,{1,1}}},
+ [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}},
+ {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}},
+ {?eh,tc_start,{surefire_SUITE,tc_ok}},
+ {?eh,tc_done,{surefire_SUITE,tc_ok,ok}},
+ {?eh,test_stats,{2,1,{1,1}}},
+ {?eh,tc_start,{surefire_SUITE,tc_fail}},
+ {?eh,tc_done,{surefire_SUITE,tc_fail,
+ {failed,{error,{test_case_failed,"this test should fail"}}}}},
+ {?eh,test_stats,{2,2,{1,1}}}] ++
+ tc_skip_events(Test,g) ++
+ [{?eh,test_stats,{2,2,{2,1}}},
+ {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
+ {?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
+ {auto_skipped,{require_failed,'_'}}}},
+ {?eh,test_stats,{2,2,{2,2}}},
+ {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}},
+ {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}],
+ [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}},
+ {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]},
+ {failed,{error,all_cases_should_be_skipped}}}},
+ {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail},
+ {failed,
+ {surefire_SUITE,init_per_group,
+ {'EXIT',all_cases_should_be_skipped}}}}},
+ {?eh,test_stats,{2,2,{2,3}}},
+ {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail},
+ {failed,
+ {surefire_SUITE,init_per_group,
+ {'EXIT',all_cases_should_be_skipped}}}}}],
+ {?eh,tc_start,{surefire_SUITE,end_per_suite}},
+ {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}].
+
+tc_skip_events(skip_case_in_spec,Group) ->
+ [{?eh,tc_user_skip,{surefire_SUITE,tc_skip_name(Group),"skipped in spec"}}];
+tc_skip_events(_Test,_Group) ->
+ [{?eh,tc_start,{surefire_SUITE,tc_skip}},
+ {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}].
+
+tc_skip_name(undefined) ->
+ tc_skip;
+tc_skip_name(Group) ->
+ {tc_skip,Group}.
test_events(fail_pre_init_per_suite) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
@@ -257,6 +290,10 @@ test_events(fail_pre_init_per_suite) ->
test_suite_events(pass_SUITE) ++
test_suite_events(fail_SUITE, {1,0,{0,1}}) ++
[{?eh,stop_logging,[]}];
+test_events(skip_suite_in_spec) ->
+ [{?eh,start_logging,'_'},{?eh,start_info,{1,1,0}}] ++
+ test_suite_events(skip_all_surefire_SUITE) ++
+ [{?eh,stop_logging,[]}];
test_events(Test) ->
[{?eh,start_logging,'_'}, {?eh,start_info,{1,1,9}}] ++
test_suite_events(Test) ++
@@ -364,6 +401,8 @@ failed_or_skipped([]) ->
events_to_result(E) ->
events_to_result(E, []).
+events_to_result([{?eh,tc_user_skip,{_Suite,all,_}}|E], Result) ->
+ events_to_result(E, [[[s]]|Result]);
events_to_result([{?eh,tc_auto_skip,{_Suite,init_per_suite,_}}|E], Result) ->
{Suite,Rest} = events_to_result1(E),
events_to_result(Rest, [[[s]|Suite]|Result]);
@@ -382,7 +421,7 @@ events_to_result1([{?eh,tc_done,{_Suite, end_per_suite,R}}|E]) ->
events_to_result1([{?eh,tc_done,{_Suite,_Case,R}}|E]) ->
{Suite,Rest} = events_to_result1(E),
{[result(R)|Suite],Rest};
-events_to_result1([{?eh,tc_auto_skip,_}|E]) ->
+events_to_result1([{?eh,Skip,_}|E]) when Skip==tc_auto_skip; Skip==tc_user_skip ->
{Suite,Rest} = events_to_result1(E),
{[[s]|Suite],Rest};
events_to_result1([_|E]) ->
diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec
new file mode 100644
index 0000000000..42df8a7d1a
--- /dev/null
+++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec
@@ -0,0 +1,2 @@
+{suites,".",surefire_SUITE}.
+{skip_cases,".",surefire_SUITE,tc_skip,"skipped in spec"}.
diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec
new file mode 100644
index 0000000000..57966328ab
--- /dev/null
+++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec
@@ -0,0 +1,2 @@
+{suites,".",[skip_all_surefire_SUITE]}.
+{skip_suites,".",skip_all_surefire_SUITE,"skipped in spec"}.
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
index 228d900545..ea8a1a5662 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
@@ -161,6 +161,7 @@ test_events(ts_if_1) ->
{?eh,tc_start,{ts_if_1_SUITE,tc4}},
{?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}},
{?eh,test_stats,{1,2,{0,1}}},
+ {?eh,tc_start,{ts_if_1_SUITE,tc5}},
{?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}},
{?eh,test_stats,{1,2,{0,2}}},
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index e926abd885..05a452b99d 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -765,23 +765,23 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
{Done,RemEvs2,length(RemEvs2)}
end;
%% end_per_group auto- or user skipped
- (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize})
when AutoOrUserSkip == tc_auto_skip;
AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G ->
case match_data(R, Reason) of
match -> false;
_ -> true
end;
({EH,#event{name=tc_user_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G ->
case match_data(R, Reason) of
match -> false;
_ -> true
@@ -1008,20 +1008,20 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
{Done,RemEvs2,length(RemEvs2)}
end;
%% end_per_group auto-or user skipped
- (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize})
+ (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize})
when AutoOrUserSkip == tc_auto_skip;
AutoOrUserSkip == tc_user_skip ->
RemEvs1 =
lists:dropwhile(
fun({EH,#event{name=tc_auto_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M, Reason == R ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R ->
false;
({EH,#event{name=tc_user_skip,
node=EvNode,
- data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M, Reason == R ->
+ data={Mod,{end_per_group,EvGroupName},Reason}}}) when
+ EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R ->
false;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
@@ -1264,10 +1264,10 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev,
io:format(Dev, "~s~p]},~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ")
end;
-log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
+log_events1([E={_EH,tc_auto_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) ->
io:format(Dev, "~s~p],~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ");
-log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) ->
+log_events1([E={_EH,tc_user_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) ->
io:format(Dev, "~s~p],~n", [Ind,E]),
log_events1(Evs, Dev, Ind--" ");
log_events1([E], Dev, Ind) ->
diff --git a/lib/dialyzer/test/abstract_SUITE.erl b/lib/dialyzer/test/abstract_SUITE.erl
index 269db3e836..0e84dfab24 100644
--- a/lib/dialyzer/test/abstract_SUITE.erl
+++ b/lib/dialyzer/test/abstract_SUITE.erl
@@ -7,7 +7,7 @@
-include_lib("common_test/include/ct.hrl").
-include("dialyzer_test_constants.hrl").
--export([suite/0, all/0, init_per_suite/0, init_per_suite/1]).
+-export([suite/0, all/0, init_per_suite/0, init_per_suite/1, end_per_suite/1]).
-export([generated_case/1]).
suite() ->
@@ -24,6 +24,10 @@ init_per_suite(Config) ->
ok -> [{dialyzer_options, []}|Config]
end.
+end_per_suite(_Config) ->
+ %% This function is required since init_per_suite/1 exists.
+ ok.
+
generated_case(Config) when is_list(Config) ->
%% Equivalent to:
%%
diff --git a/lib/ssl/test/ssl_npn_hello_SUITE.erl b/lib/ssl/test/ssl_npn_hello_SUITE.erl
index 69aeea10c5..0b1de1dc1c 100644
--- a/lib/ssl/test/ssl_npn_hello_SUITE.erl
+++ b/lib/ssl/test/ssl_npn_hello_SUITE.erl
@@ -50,6 +50,10 @@ init_per_suite(Config) ->
{skip, "Crypto did not start"}
end.
+end_per_suite(_Config) ->
+ %% This function is required since init_per_suite/1 exists.
+ ok.
+
init_per_testcase(_TestCase, Config) ->
ssl_test_lib:ct_log_supported_protocol_versions(Config),
ct:timetrap({seconds, 5}),