aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/doc/src/notes.xml62
-rw-r--r--lib/common_test/src/ct_hooks.erl11
-rw-r--r--lib/common_test/src/ct_release_test.erl22
-rw-r--r--lib/common_test/src/test_server.erl20
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl54
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl49
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl81
-rw-r--r--lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl4
8 files changed, 280 insertions, 23 deletions
diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml
index a64818da7b..c454608bbe 100644
--- a/lib/common_test/doc/src/notes.xml
+++ b/lib/common_test/doc/src/notes.xml
@@ -62,6 +62,37 @@
</section>
+<section><title>Common_Test 1.17.2.1</title>
+
+ <section><title>Fixed Bugs and Malfunctions</title>
+ <list>
+ <item>
+ <p>
+ If a ct hook is installed in the <c>suite/0</c> function
+ in a test suite, then the hook's <c>terminate/1</c>
+ function would be called several times without it's
+ <c>init/2</c> function being called first. This is now
+ corrected.</p>
+ <p>
+ Own Id: OTP-15863 Aux Id: ERIERL-370 </p>
+ </item>
+ <item>
+ <p>
+ If <c>init_per_testcase</c> fails, the test itself is
+ skipped. According to the documentation, it should be
+ possible to change the result to failed in a hook
+ function. The only available hook function in this case
+ is <c>post_init_per_testcase</c>, but changing the return
+ value there did not affect the test case result. This is
+ now corrected.</p>
+ <p>
+ Own Id: OTP-15869 Aux Id: ERIERL-350 </p>
+ </item>
+ </list>
+ </section>
+
+</section>
+
<section><title>Common_Test 1.17.2</title>
<section><title>Fixed Bugs and Malfunctions</title>
@@ -239,6 +270,37 @@
</section>
+<section><title>Common_Test 1.15.4.3</title>
+
+ <section><title>Fixed Bugs and Malfunctions</title>
+ <list>
+ <item>
+ <p>
+ If a ct hook is installed in the <c>suite/0</c> function
+ in a test suite, then the hook's <c>terminate/1</c>
+ function would be called several times without it's
+ <c>init/2</c> function being called first. This is now
+ corrected.</p>
+ <p>
+ Own Id: OTP-15863 Aux Id: ERIERL-370 </p>
+ </item>
+ <item>
+ <p>
+ If <c>init_per_testcase</c> fails, the test itself is
+ skipped. According to the documentation, it should be
+ possible to change the result to failed in a hook
+ function. The only available hook function in this case
+ is <c>post_init_per_testcase</c>, but changing the return
+ value there did not affect the test case result. This is
+ now corrected.</p>
+ <p>
+ Own Id: OTP-15869 Aux Id: ERIERL-350 </p>
+ </item>
+ </list>
+ </section>
+
+</section>
+
<section><title>Common_Test 1.15.4.2</title>
<section><title>Fixed Bugs and Malfunctions</title>
diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl
index 97c349578f..94551d6815 100644
--- a/lib/common_test/src/ct_hooks.erl
+++ b/lib/common_test/src/ct_hooks.erl
@@ -363,7 +363,16 @@ terminate_if_scope_ends(HookId, Function0, Hooks) ->
Function = strip_config(Function0),
case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of
#ct_hook_config{ id = HookId, scope = Function} = Hook ->
- terminate([Hook]),
+ case Function of
+ [AllOrGroup,_] when AllOrGroup=:=post_all;
+ AllOrGroup=:=post_groups ->
+ %% The scope only contains one function (post_all
+ %% or post_groups), and init has not been called,
+ %% so skip terminate as well.
+ ok;
+ _ ->
+ terminate([Hook])
+ end,
lists:keydelete(HookId, #ct_hook_config.id, Hooks);
_ ->
Hooks
diff --git a/lib/common_test/src/ct_release_test.erl b/lib/common_test/src/ct_release_test.erl
index ac3dcab7c9..839fb300c7 100644
--- a/lib/common_test/src/ct_release_test.erl
+++ b/lib/common_test/src/ct_release_test.erl
@@ -475,7 +475,7 @@ fetch_all_apps(Node) ->
A = list_to_atom(filename:basename(filename:rootname(F))),
_ = rpc:call(Node,application,load,[A]),
case rpc:call(Node,application,get_key,[A,vsn]) of
- {ok,V} -> [{A,V}];
+ {ok,V} -> [{A,V,rpc:call(Node,code,lib_dir,[A])}];
_ -> []
end
end,
@@ -517,7 +517,7 @@ upgrade(Apps,Level,Callback,CreateDir,InstallDir,Config) ->
target_system(Apps,CreateDir,InstallDir,{FromVsn,_,AllAppsVsns,Path}) ->
RelName0 = "otp-"++FromVsn,
- AppsVsns = [{A,V} || {A,V} <- AllAppsVsns, lists:member(A,Apps)],
+ AppsVsns = [{A,V,D} || {A,V,D} <- AllAppsVsns, lists:member(A,Apps)],
{RelName,ErtsVsn} = create_relfile(AppsVsns,CreateDir,RelName0,FromVsn),
%% Create .script and .boot
@@ -636,8 +636,7 @@ do_upgrade({Cb,InitState},FromVsn,FromAppsVsns,ToRel,ToAppsVsns,InstallDir) ->
{ok,Node} = start_node(Start,FromVsn,FromAppsVsns),
ct:log("Node started: ~p",[Node]),
- CtData = #ct_data{from = [{A,V,code:lib_dir(A)} || {A,V} <- FromAppsVsns],
- to=[{A,V,code:lib_dir(A)} || {A,V} <- ToAppsVsns]},
+ CtData = #ct_data{from = FromAppsVsns,to=ToAppsVsns},
State1 = do_callback(Node,Cb,upgrade_init,[CtData,InitState]),
[{"OTP upgrade test",FromVsn,_,permanent}] =
@@ -724,14 +723,14 @@ previous_major(Rel) ->
integer_to_list(list_to_integer(Rel)-1).
create_relfile(AppsVsns,CreateDir,RelName0,RelVsn) ->
- UpgradeAppsVsns = [{A,V,restart_type(A)} || {A,V} <- AppsVsns],
+ UpgradeAppsVsns = [{A,V,restart_type(A)} || {A,V,_D} <- AppsVsns],
CoreAppVsns0 = get_vsns([kernel,stdlib,sasl]),
CoreAppVsns =
- [{A,V,restart_type(A)} || {A,V} <- CoreAppVsns0,
+ [{A,V,restart_type(A)} || {A,V,_D} <- CoreAppVsns0,
false == lists:keymember(A,1,AppsVsns)],
- Apps = [App || {App,_} <- AppsVsns],
+ Apps = [App || {App,_,_} <- AppsVsns],
StartDepsVsns = get_start_deps(Apps,CoreAppVsns),
StartApps = [StartApp || {StartApp,_,_} <- StartDepsVsns] ++ Apps,
@@ -744,7 +743,7 @@ create_relfile(AppsVsns,CreateDir,RelName0,RelVsn) ->
%% processes of these applications will not be running.
TestToolAppsVsns0 = get_vsns([common_test]),
TestToolAppsVsns =
- [{A,V,none} || {A,V} <- TestToolAppsVsns0,
+ [{A,V,none} || {A,V,_D} <- TestToolAppsVsns0,
false == lists:keymember(A,1,AllAppsVsns0)],
AllAppsVsns1 = AllAppsVsns0 ++ TestToolAppsVsns,
@@ -766,7 +765,7 @@ get_vsns(Apps) ->
[begin
_ = application:load(A),
{ok,V} = application:get_key(A,vsn),
- {A,V}
+ {A,V,code:lib_dir(A)}
end || A <- Apps].
get_start_deps([App|Apps],Acc) ->
@@ -880,8 +879,9 @@ start_node(Start,ExpVsn,ExpAppsVsns) ->
erlang:port_close(Port),
wait_node_up(permanent,ExpVsn,ExpAppsVsns).
-wait_node_up(ExpStatus,ExpVsn,ExpAppsVsns) ->
+wait_node_up(ExpStatus,ExpVsn,ExpAppsVsns0) ->
Node = node_name(?testnode),
+ ExpAppsVsns = [{A,V} || {A,V,_D} <- ExpAppsVsns0],
wait_node_up(Node,ExpStatus,ExpVsn,lists:keysort(1,ExpAppsVsns),60).
wait_node_up(Node,ExpStatus,ExpVsn,ExpAppsVsns,0) ->
@@ -893,7 +893,7 @@ wait_node_up(Node,ExpStatus,ExpVsn,ExpAppsVsns,N) ->
rpc:call(Node, application, which_applications, [])} of
{[{_,ExpVsn,_,_}],Apps} when is_list(Apps) ->
case [{A,V} || {A,_,V} <- lists:keysort(1,Apps),
- lists:keymember(A,1,ExpAppsVsns)] of
+ lists:keymember(A,1,ExpAppsVsns)] of
ExpAppsVsns ->
{ok,Node};
_ ->
diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl
index 756cd4d692..588396f101 100644
--- a/lib/common_test/src/test_server.erl
+++ b/lib/common_test/src/test_server.erl
@@ -1364,23 +1364,29 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->
{NOk,_} when NOk == auto_skip; NOk == fail;
NOk == skip ; NOk == skipped ->
{_,Args} = Res,
- IPTCEndRes =
+ {NewConfig,IPTCEndRes} =
case do_end_tc_call1(Mod, IPTC, Res, Return) of
IPTCEndConfig when is_list(IPTCEndConfig) ->
- IPTCEndConfig;
+ {IPTCEndConfig,IPTCEndConfig};
+ {failed,RetReason} when Return=:={fail,RetReason} ->
+ %% Fail reason not changed by framework or hook
+ {Args,Return};
+ {SF,_} = IPTCEndResult when SF=:=skip; SF=:=skipped;
+ SF=:=fail; SF=:=failed ->
+ {Args,IPTCEndResult};
_ ->
- Args
+ {Args,Return}
end,
EPTCInitRes =
case do_init_tc_call(Mod,{end_per_testcase_not_run,Func},
- IPTCEndRes,Return) of
+ NewConfig,IPTCEndRes) of
{ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->
- {Return,EPTCInitConfig};
+ {IPTCEndRes,EPTCInitConfig};
_ ->
- {Return,IPTCEndRes}
+ {IPTCEndRes,NewConfig}
end,
do_end_tc_call1(Mod, {end_per_testcase_not_run,Func},
- EPTCInitRes, Return);
+ EPTCInitRes, IPTCEndRes);
_Ok ->
do_end_tc_call1(Mod, IPTC, Res, Return)
end;
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 340b8f3d52..b87464f5e4 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -86,7 +86,7 @@ all(suite) ->
scope_suite_state_cth,
fail_pre_suite_cth, double_fail_pre_suite_cth,
fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth,
- skip_pre_init_tc_cth,
+ skip_pre_init_tc_cth, fail_post_init_tc_cth,
skip_post_suite_cth, recover_post_suite_cth, update_config_cth,
state_update_cth, update_result_cth, options_cth, same_id_cth,
fail_n_skip_with_minimal_cth, prio_cth, no_config,
@@ -206,6 +206,10 @@ skip_pre_init_tc_cth(Config) ->
do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl",
[skip_pre_init_tc_cth],Config).
+fail_post_init_tc_cth(Config) ->
+ do_test(fail_post_init_tc_cth, "ct_fail_init_tc_SUITE.erl",
+ [fail_post_init_tc_cth],Config).
+
recover_post_suite_cth(Config) when is_list(Config) ->
do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl",
[recover_post_suite_cth],Config).
@@ -671,9 +675,15 @@ test_events(scope_suite_cth) ->
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
%% check that post_groups and post_all comes before init when hook
%% is installed in suite/0
+ %% And there should be no terminate after these, since init is
+ %% not yet called.
{?eh,cth,{'_',post_groups,['_',[]]}},
- {?eh,cth,{'_',post_all,['_','_',[]]}},
- {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}},
+ {negative,
+ {?eh,cth,{'_',terminate,['_']}},
+ {?eh,cth,{'_',post_all,['_','_',[]]}}},
+ {negative,
+ {?eh,cth,{'_',terminate,['_']}},
+ {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}}},
{?eh,cth,{'_',id,[[]]}},
{?eh,cth,{'_',init,['_',[]]}},
{?eh,cth,{'_',pre_init_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}},
@@ -1036,6 +1046,44 @@ test_events(skip_pre_init_tc_cth) ->
{?eh,stop_logging,[]}
];
+test_events(fail_post_init_tc_cth) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,init,['_',[]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,tc_start,{ct_fail_init_tc_SUITE,init_per_suite}},
+ {?eh,cth,{empty_cth,pre_init_per_suite,[ct_fail_init_tc_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_suite,
+ [ct_fail_init_tc_SUITE,'$proplist','$proplist',[]]}},
+ {?eh,tc_done,{ct_fail_init_tc_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{ct_fail_init_tc_SUITE,test_case}},
+ {?eh,cth,{empty_cth,pre_init_per_testcase,
+ [ct_fail_init_tc_SUITE,test_case,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_init_per_testcase,
+ [ct_fail_init_tc_SUITE,test_case,'$proplist',
+ {skip,
+ {failed,
+ {ct_fail_init_tc_SUITE,init_per_testcase,
+ {{test_case_failed,"Failed in init_per_testcase"},'_'}}}},
+ []]}},
+ {?eh,tc_done,{ct_fail_init_tc_SUITE,test_case,
+ {failed,"Changed skip to fail in post_init_per_testcase"}}},
+ {?eh,cth,{empty_cth,on_tc_fail,
+ [ct_fail_init_tc_SUITE,test_case,
+ "Changed skip to fail in post_init_per_testcase",
+ []]}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{ct_fail_init_tc_SUITE,end_per_suite}},
+ {?eh,cth,{empty_cth,pre_end_per_suite,[ct_fail_init_tc_SUITE,'$proplist',[]]}},
+ {?eh,cth,{empty_cth,post_end_per_suite,
+ [ct_fail_init_tc_SUITE,'$proplist',ok,[]]}},
+ {?eh,tc_done,{ct_fail_init_tc_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(recover_post_suite_cth) ->
Suite = ct_cth_fail_per_suite_SUITE,
[
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl
new file mode 100644
index 0000000000..96ddfc5782
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl
@@ -0,0 +1,49 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_fail_init_tc_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(TestCase, _Config) ->
+ ct:fail("Failed in init_per_testcase").
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
+
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl
new file mode 100644
index 0000000000..ca9f05c40f
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl
@@ -0,0 +1,81 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(fail_post_init_tc_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_init_per_testcase(Suite,TC,Config,{skip,_}=Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State),
+ {{fail,"Changed skip to fail in post_init_per_testcase"},State};
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC, Reason, State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl b/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl
index 7f0ba65791..fe69ad0748 100644
--- a/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl
+++ b/lib/common_test/test/ct_release_test_SUITE_data/release_test_SUITE.erl
@@ -47,7 +47,9 @@ end_per_suite(_Config) ->
init_per_testcase(major_fail_no_init, Config) ->
Config;
init_per_testcase(_Case, Config) ->
- ct_release_test:init(Config).
+ Config1 = ct_release_test:init(Config),
+ ct:log("ct_release_test:init/1 returned:~n~p",[Config1]),
+ Config1.
end_per_testcase(_Case, Config) ->
ct_release_test:cleanup(Config).
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-16 05:09:51 +0000