From 6b7463094db2ec3c2c0e4f8784dcd64ef186c96b Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Thu, 16 Feb 2017 15:38:05 +0100 Subject: [ct] Fix faulty hook callbacks for skipped tests The following bugs are corrected: * Tests that were skipped before calling pre_init_per_* got faulty calls to the corresponding post_init_per_*. E.g. if a test was skipped because suite/0 returned a 'require' statement that was not fulfilled, then post_init_per_suite would be called, even though pre_init_per_suite and init_per_suite were not called. * Tests that were skipped before or in init_per_testcase got faulty calls to pre_end_per_testcase and post_end_per_testcase (end_per_testcase is not called in these situations). Test are added to make sure that the expected callbacks, and only those, are called when tests are skipped in different ways. Conflicts: lib/common_test/test/ct_hooks_SUITE.erl --- .../cth/tests/skip_case_SUITE.erl | 87 ++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl (limited to 'lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl') diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl new file mode 100644 index 0000000000..436d496c32 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl @@ -0,0 +1,87 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_case_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(skip_in_init,Config) -> + {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(fail_in_init,Config) -> + ct:fail("Failed in init_per_testcase/2"); +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [skip_in_spec, + skip_in_init, + fail_in_init, + skip_in_case, + req_auto_skip, + fail_auto_skip + ]. + +%% Test cases starts here. +skip_in_spec(Config) -> + ct:fail("This test shall never be run. " + "It shall be skipped in the test spec."). + +skip_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall be skipped in init_per_testcase/2."). + +fail_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall fail in init_per_testcase/2."). + +skip_in_case(Config) -> + {skip,"Skipped in test case function"}. + +req_auto_skip() -> + [{require,whatever}]. +req_auto_skip(Config) -> + ct:fail("This test shall never be run due to " + "failed require"). + +fail_auto_skip() -> + faulty_return_value. +fail_auto_skip(Config) -> + ct:fail("This test shall never be run due to " + "faulty return from info function"). -- cgit v1.2.3 From 827732949ba9cd844347bca1a46b2e2055810e69 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Thu, 16 Feb 2017 12:16:21 +0100 Subject: [ct] Correctly handle process died in init and end_per_testcase Documentation says that a test case is skipped if init_per_testcase fails. This is correct if init_per_testcase fails in a normal way, but if it gets an exit signal causing the process to die, then the case was earlier marked as failed with reason {skip,...}. This is now corrected, so the case is auto skipped. If end_per_testcase crashes, it does not change the result of a test case, it only adds a warning in the comment field. But if it gets an exit signal causing the process to die while running end_per_testcase, then the case was earlier marked as failed in addition to this warning. This is now corrected, so the result of the test case is not changed by an exit signal during end_per_testcase. --- .../ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl') diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl index 436d496c32..dad80ae914 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl @@ -43,9 +43,15 @@ init_per_testcase(skip_in_init,Config) -> {skip,"Skipped in init_per_testcase/2"}; init_per_testcase(fail_in_init,Config) -> ct:fail("Failed in init_per_testcase/2"); +init_per_testcase(exit_in_init,Config) -> + exit(self(),"Exit in init_per_testcase/2"); init_per_testcase(_,Config) -> Config. +end_per_testcase(fail_in_end,_) -> + ct:fail("Failed in end_per_testcase/2"); +end_per_testcase(exit_in_end,_) -> + exit(self(),"Exit in end_per_testcase/2"); end_per_testcase(_,_) -> ok. @@ -53,6 +59,9 @@ all() -> [skip_in_spec, skip_in_init, fail_in_init, + exit_in_init, + fail_in_end, + exit_in_end, skip_in_case, req_auto_skip, fail_auto_skip @@ -71,6 +80,16 @@ fail_in_init(Config) -> ct:fail("This test shall never be run. " "It shall fail in init_per_testcase/2."). +exit_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall exit in init_per_testcase/2."). + +fail_in_end(Config) -> + ok. + +exit_in_end(Config) -> + ok. + skip_in_case(Config) -> {skip,"Skipped in test case function"}. -- cgit v1.2.3