From 356d1147ad9fc120b0750b886678ef92eb6c0985 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Mon, 20 May 2019 15:38:26 +0200 Subject: [ct] Allow post_init_per_testcase to change skip to fail --- lib/common_test/src/test_server.erl | 20 ++++-- lib/common_test/test/ct_hooks_SUITE.erl | 44 +++++++++++- .../cth/tests/ct_fail_init_tc_SUITE.erl | 49 +++++++++++++ .../cth/tests/fail_post_init_tc_cth.erl | 81 ++++++++++++++++++++++ 4 files changed, 186 insertions(+), 8 deletions(-) create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl create mode 100644 lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 756cd4d692..588396f101 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -1364,23 +1364,29 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) -> {NOk,_} when NOk == auto_skip; NOk == fail; NOk == skip ; NOk == skipped -> {_,Args} = Res, - IPTCEndRes = + {NewConfig,IPTCEndRes} = case do_end_tc_call1(Mod, IPTC, Res, Return) of IPTCEndConfig when is_list(IPTCEndConfig) -> - IPTCEndConfig; + {IPTCEndConfig,IPTCEndConfig}; + {failed,RetReason} when Return=:={fail,RetReason} -> + %% Fail reason not changed by framework or hook + {Args,Return}; + {SF,_} = IPTCEndResult when SF=:=skip; SF=:=skipped; + SF=:=fail; SF=:=failed -> + {Args,IPTCEndResult}; _ -> - Args + {Args,Return} end, EPTCInitRes = case do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, - IPTCEndRes,Return) of + NewConfig,IPTCEndRes) of {ok,EPTCInitConfig} when is_list(EPTCInitConfig) -> - {Return,EPTCInitConfig}; + {IPTCEndRes,EPTCInitConfig}; _ -> - {Return,IPTCEndRes} + {IPTCEndRes,NewConfig} end, do_end_tc_call1(Mod, {end_per_testcase_not_run,Func}, - EPTCInitRes, Return); + EPTCInitRes, IPTCEndRes); _Ok -> do_end_tc_call1(Mod, IPTC, Res, Return) end; diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 08c18d91e2..b87464f5e4 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -86,7 +86,7 @@ all(suite) -> scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, - skip_pre_init_tc_cth, + skip_pre_init_tc_cth, fail_post_init_tc_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, state_update_cth, update_result_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, @@ -206,6 +206,10 @@ skip_pre_init_tc_cth(Config) -> do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl", [skip_pre_init_tc_cth],Config). +fail_post_init_tc_cth(Config) -> + do_test(fail_post_init_tc_cth, "ct_fail_init_tc_SUITE.erl", + [fail_post_init_tc_cth],Config). + recover_post_suite_cth(Config) when is_list(Config) -> do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl", [recover_post_suite_cth],Config). @@ -1042,6 +1046,44 @@ test_events(skip_pre_init_tc_cth) -> {?eh,stop_logging,[]} ]; +test_events(fail_post_init_tc_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_fail_init_tc_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[ct_fail_init_tc_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_fail_init_tc_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_fail_init_tc_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_fail_init_tc_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_fail_init_tc_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [ct_fail_init_tc_SUITE,test_case,'$proplist', + {skip, + {failed, + {ct_fail_init_tc_SUITE,init_per_testcase, + {{test_case_failed,"Failed in init_per_testcase"},'_'}}}}, + []]}}, + {?eh,tc_done,{ct_fail_init_tc_SUITE,test_case, + {failed,"Changed skip to fail in post_init_per_testcase"}}}, + {?eh,cth,{empty_cth,on_tc_fail, + [ct_fail_init_tc_SUITE,test_case, + "Changed skip to fail in post_init_per_testcase", + []]}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{ct_fail_init_tc_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[ct_fail_init_tc_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_fail_init_tc_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_fail_init_tc_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(recover_post_suite_cth) -> Suite = ct_cth_fail_per_suite_SUITE, [ diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl new file mode 100644 index 0000000000..96ddfc5782 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_fail_init_tc_SUITE.erl @@ -0,0 +1,49 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_fail_init_tc_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(TestCase, _Config) -> + ct:fail("Failed in init_per_testcase"). + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) when is_list(Config) -> + ok. + diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl new file mode 100644 index 0000000000..ca9f05c40f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_init_tc_cth.erl @@ -0,0 +1,81 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(fail_post_init_tc_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,{skip,_}=Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State), + {{fail,"Changed skip to fail in post_init_per_testcase"},State}; +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). -- cgit v1.2.3 From 99535d6bbbb557e86af5c074718da5223cf934d9 Mon Sep 17 00:00:00 2001 From: Micael Karlberg Date: Fri, 14 Jun 2019 11:41:49 +0200 Subject: [snmp|test] Issues with netsnmp test on Solaris 5.11 On solaris 5.11 (actually OpenIndiana Hipster) the arguments for (some of) the netsnmp programs differ from what is used in our test suite. Added actual checks for *one of* the failures, which is indicated with the printout "illegal option". In those cases the test case is instead skipped. --- lib/snmp/test/snmp_to_snmpnet_SUITE.erl | 143 +++++++++++++++++++++----------- 1 file changed, 94 insertions(+), 49 deletions(-) diff --git a/lib/snmp/test/snmp_to_snmpnet_SUITE.erl b/lib/snmp/test/snmp_to_snmpnet_SUITE.erl index b83c7461d1..b8bdb30271 100644 --- a/lib/snmp/test/snmp_to_snmpnet_SUITE.erl +++ b/lib/snmp/test/snmp_to_snmpnet_SUITE.erl @@ -247,9 +247,17 @@ erlang_agent_netsnmp_get(Config) when is_list(Config) -> start_agent(Config), Oid = ?sysDescr_instance, Expected = expected(Oid, get), - [Expected = snmpget(Oid, Transport, Config) - || Transport <- Transports], - ok. + try + begin + [Expected = snmpget(Oid, Transport, Config) + || Transport <- Transports], + ok + end + catch + throw:{skip, _} = SKIP -> + SKIP + end. + %%-------------------------------------------------------------------- erlang_manager_netsnmp_get() -> @@ -260,29 +268,34 @@ erlang_manager_netsnmp_get(Config) when is_list(Config) -> SysDescr = "Net-SNMP agent", TargetName = "Target Net-SNMP agent", Transports = ?config(transports, Config), - ProgHandle = start_snmpd(Community, SysDescr, Config), - start_manager(Config), - snmp_manager_user:start_link(self(), test_user), - [snmp_manager_user:register_agent( - TargetName++domain_suffix(Domain), - [{reg_type, target_name}, - {tdomain, Domain}, {taddress, Addr}, - {community, Community}, {engine_id, "EngineId"}, - {version, v2}, {sec_model, v2c}, {sec_level, noAuthNoPriv}]) - || {Domain, Addr} <- Transports], - Results = - [snmp_manager_user:sync_get( - TargetName++domain_suffix(Domain), - [?sysDescr_instance]) - || {Domain, _} <- Transports], - ct:pal("sync_get -> ~p", [Results]), - snmp_manager_user:stop(), - stop_program(ProgHandle), - [{ok, - {noError, 0, - [{varbind, ?sysDescr_instance, 'OCTET STRING', SysDescr,1}] }, - _} = R || R <- Results], - ok. + case start_snmpd(Community, SysDescr, Config) of + {skip, _} = SKIP -> + SKIP; + ProgHandle -> + start_manager(Config), + snmp_manager_user:start_link(self(), test_user), + [snmp_manager_user:register_agent( + TargetName++domain_suffix(Domain), + [{reg_type, target_name}, + {tdomain, Domain}, {taddress, Addr}, + {community, Community}, {engine_id, "EngineId"}, + {version, v2}, {sec_model, v2c}, {sec_level, noAuthNoPriv}]) + || {Domain, Addr} <- Transports], + Results = + [snmp_manager_user:sync_get( + TargetName++domain_suffix(Domain), + [?sysDescr_instance]) + || {Domain, _} <- Transports], + ct:pal("sync_get -> ~p", [Results]), + snmp_manager_user:stop(), + stop_program(ProgHandle), + [{ok, + {noError, 0, + [{varbind, ?sysDescr_instance, 'OCTET STRING', SysDescr,1}] }, + _} = R || R <- Results], + ok + end. + %%-------------------------------------------------------------------- erlang_agent_netsnmp_inform(Config) when is_list(Config) -> @@ -292,17 +305,19 @@ erlang_agent_netsnmp_inform(Config) when is_list(Config) -> start_agent(Config), ok = snmpa:load_mib(snmp_master_agent, filename:join(DataDir, Mib)), - ProgHandle = start_snmptrapd(Mib, Config), - - snmpa:send_notification( - snmp_master_agent, testTrapv22, {erlang_agent_test, self()}), - - receive - {snmp_targets, erlang_agent_test, Addresses} -> - ct:pal("Notification sent to: ~p~n", [Addresses]), - erlang_agent_netsnmp_inform_responses(Addresses) - end, - stop_program(ProgHandle). + case start_snmptrapd(Mib, Config) of + {skip, _} = SKIP -> + SKIP; + ProgHandle -> + snmpa:send_notification( + snmp_master_agent, testTrapv22, {erlang_agent_test, self()}), + receive + {snmp_targets, erlang_agent_test, Addresses} -> + ct:pal("Notification sent to: ~p~n", [Addresses]), + erlang_agent_netsnmp_inform_responses(Addresses) + end, + stop_program(ProgHandle) + end. erlang_agent_netsnmp_inform_responses([]) -> receive @@ -326,6 +341,7 @@ erlang_agent_netsnmp_inform_responses([Address | Addresses]) -> %%-------------------------------------------------------------------- %% Internal functions ------------------------------------------------ %%-------------------------------------------------------------------- + snmpget(Oid, Transport, Config) -> Versions = ?config(snmp_versions, Config), @@ -335,10 +351,14 @@ snmpget(Oid, Transport, Config) -> "-Cf", net_snmp_addr_str(Transport), oid_str(Oid)], - ProgHandle = start_program(snmpget, Args, none, Config), - {_, line, Line} = get_program_output(ProgHandle), - stop_program(ProgHandle), - Line. + case start_program(snmpget, Args, none, Config) of + {skip, _} = SKIP -> + throw(SKIP); + ProgHandle -> + {_, line, Line} = get_program_output(ProgHandle), + stop_program(ProgHandle), + Line + end. start_snmptrapd(Mibs, Config) -> DataDir = ?config(data_dir, Config), @@ -382,12 +402,13 @@ start_program(Prog, Args, StartCheckMP, Config) -> DataDir = ?config(data_dir, Config), StartWrapper = filename:join(DataDir, "start_stop_wrapper"), Parent = self(), - Pid = - spawn_link( + %% process_flag(trap_exit, true), + {Pid, Mon} = + spawn_monitor( fun () -> run_program(Parent, StartWrapper, [Path | Args]) end), - start_check(Pid, erlang:monitor(process, Pid), StartCheckMP). + start_check(Pid, Mon, StartCheckMP). start_check(Pid, Mon, none) -> {Pid, Mon}; @@ -400,6 +421,10 @@ start_check(Pid, Mon, StartCheckMP) -> nomatch -> start_check(Pid, Mon, StartCheckMP) end; + {'DOWN', Mon, _, _Pid, {skip, Reason} = SKIP} -> + ct:pal("Received DOWN from ~p" + "~n Skip Reason: ~p", [_Pid, Reason]), + SKIP; {'DOWN', Mon, _, _, Reason} -> ct:fail("Prog ~p start failed: ~p", [Pid, Reason]) end. @@ -446,14 +471,34 @@ run_program_loop(Parent, Port, Buf) -> {Port, {data, {Flag, Data}}} -> case Flag of eol -> - Line = iolist_to_binary(lists:reverse(Buf, Data)), - ct:pal("Prog ~p output: ~s", [Port, Line]), - Parent ! {self(), line, Line}, - run_program_loop(Parent, Port, []); + Line = iolist_to_binary(lists:reverse(Buf, Data)), + ct:pal("Prog ~p output: ~s", [Port, Line]), + %% There are potentially many different fail outputs, + %% but for now we test for just this one: illegal option + IOpt = "illegal option", + case string:find(binary_to_list(Line), IOpt) of + nomatch -> + Parent ! {self(), line, Line}, + run_program_loop(Parent, Port, []); + Line2 -> + %% Try to extract the actual illegal option string + IOpt2 = + case string:take( + string:prefix(Line2, IOpt), [$-, $ ]) of + {_, Str} when length(Str) > 0 -> + Str; + _X -> + Line2 + end, + ct:pal("Force program ~p stop", [Port]), + true = port_command(Port, <<"stop\n">>), + (catch port_close(Port)), + exit({skip, {illegal_option, IOpt2}}) + end; noeol -> run_program_loop(Parent, Port, [Data | Buf]) end; - {Port, {exit_status,ExitStatus}} -> + {Port, {exit_status, ExitStatus}} -> ct:pal("Prog ~p exit: ~p", [Port, ExitStatus]), catch port_close(Port), Parent ! {self(), exit, ExitStatus}; -- cgit v1.2.3