diff options
Diffstat (limited to 'lib/common_test/test')
47 files changed, 2409 insertions, 202 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index ecd1f727a2..e510b74d6a 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -74,7 +74,9 @@ MODULES= \ ct_SUITE \ ct_keep_logs_SUITE \ ct_unicode_SUITE \ - ct_auto_clean_SUITE + ct_auto_clean_SUITE \ + ct_util_SUITE \ + ct_tc_repeat_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_auto_clean_SUITE.erl b/lib/common_test/test/ct_auto_clean_SUITE.erl index fd81430d0d..a89c90eb79 100644 --- a/lib/common_test/test/ct_auto_clean_SUITE.erl +++ b/lib/common_test/test/ct_auto_clean_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl b/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl index dae7c1e22c..e779f70693 100644 --- a/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl +++ b/lib/common_test/test/ct_auto_clean_SUITE_data/ac_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl b/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl index 137c81969d..3f8d3957cc 100644 --- a/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl +++ b/lib/common_test/test/ct_auto_clean_SUITE_data/cth_auto_clean.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/common_test/test/ct_auto_compile_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE.erl index dface99b8f..f88f13c889 100644 --- a/lib/common_test/test/ct_auto_compile_SUITE.erl +++ b/lib/common_test/test/ct_auto_compile_SUITE.erl @@ -169,7 +169,7 @@ test_events(ac_flag) -> {?eh,start_info,{1,1,3}}, {?eh,tc_start,{ct_framework,error_in_suite}}, {?eh,tc_done,{ct_framework,error_in_suite, - {failed,{error,'bad_SUITE can not be compiled or loaded'}}}}, + {failed,{error,'bad_SUITE cannot be compiled or loaded'}}}}, {?eh,tc_start,{dummy_SUITE,init_per_suite}}, {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, {?eh,test_stats,{1,1,{1,0}}}, @@ -186,7 +186,7 @@ test_events(ac_spec) -> {?eh,start_info,{1,1,3}}, {?eh,tc_start,{ct_framework,error_in_suite}}, {?eh,tc_done,{ct_framework,error_in_suite, - {failed,{error,'bad_SUITE can not be compiled or loaded'}}}}, + {failed,{error,'bad_SUITE cannot be compiled or loaded'}}}}, {?eh,tc_start,{dummy_SUITE,init_per_suite}}, {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, {?eh,test_stats,{1,1,{1,0}}}, diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 250700741c..5ffc735d6a 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -213,8 +213,8 @@ reformat_events(Events, EH) -> skip_dynamic() -> case os:getenv("TS_EXTRA_PLATFORM_LABEL") of TSExtraPlatformLabel when is_list(TSExtraPlatformLabel) -> - case string:str(TSExtraPlatformLabel,"TimeWarpingOS") of - 0 -> false; + case string:find(TSExtraPlatformLabel,"TimeWarpingOS") of + nomatch -> false; _ -> true end; _ -> diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl index 4e013b8056..e9e8b2a54d 100644 --- a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,9 +23,7 @@ %% Description: %% This file contains the test cases for the code coverage support %% -%% @author Support -%% @doc Test of code coverage support in common_test -%% @end +%% Test of code coverage support in common_test %%---------------------------------------------------------------------- %%---------------------------------------------------------------------- -module(cover_SUITE). diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 7468ebe9d9..d31bd26273 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -648,33 +648,35 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, - {?eh,test_stats,{13,8,{0,19}}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc1, + {auto_skipped, + {failed,{ct_framework,init_tc,{timetrap,500}}}}}}, + {?eh,test_stats,{13,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,8,{0,19}}}, + {?eh,test_stats,{14,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,8,{0,19}}}, + {?eh,test_stats,{15,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,8,{0,19}}}, + {?eh,test_stats,{16,7,{0,20}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,8,{0,19}}}, + {?eh,test_stats,{17,7,{0,20}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,8,{0,19}}}, + {?eh,test_stats,{18,7,{0,20}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, @@ -728,25 +730,30 @@ test_events(lib_error) -> {lib_error_1_SUITE,no_lines_throw,{failed,{error,{thrown,catch_me_if_u_can}}}}}, {?eh,test_stats,{0,8,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_error}}, - {?eh,tc_done,{ct_framework,init_tc, - {framework_error,{{badmatch,[1,2]},'_'}}}}, - {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_error, + {auto_skipped, + {failed, + {ct_framework,init_tc, + {{badmatch,[1,2]},'_'}}}}}}, + {?eh,test_stats,{0,8,{0,1}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_exit}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,byebye}}}, - {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_exit, + {auto_skipped,{failed,{ct_framework,init_tc,byebye}}}}}, + {?eh,test_stats,{0,8,{0,2}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_throw}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,catch_me_if_u_can}}}, - {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_throw, + {auto_skipped,{failed,{ct_framework,init_tc, + catch_me_if_u_can}}}}}, + {?eh,test_stats,{0,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_error}}, - {?eh,tc_done,{ct_framework,end_tc, - {framework_error,{{badmatch,[1,2]},'_'}}}}, - {?eh,test_stats,{0,12,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_error,ok}}, % warning in comment + {?eh,test_stats,{1,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_exit}}, - {?eh,tc_done,{ct_framework,end_tc,{framework_error,byebye}}}, - {?eh,test_stats,{0,13,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_exit,ok}}, % warning in comment + {?eh,test_stats,{2,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_throw}}, - {?eh,tc_done,{ct_framework,end_tc,{framework_error,catch_me_if_u_can}}}, - {?eh,test_stats,{0,14,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_throw,ok}}, % warning in comment + {?eh,test_stats,{3,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_per_suite}}, {?eh,tc_done,{lib_error_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl index 530ee09654..ef8c05cb2e 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,9 +18,9 @@ %% %CopyrightEnd% %% -%%% @doc Common Test Example Suite Callback module. +%%% Common Test Example Suite Callback module. %%% -%%% <p>This module gives an example of a common test CTH (Common Test Hook). +%%% This module gives an example of a common test CTH (Common Test Hook). %%% There are many ways to add a CTH to a test run, you can do it either in %%% the command line using -ct_hook, in a test spec using %%% {ct_hook,M} or in the suite it self by returning ct_hook @@ -31,7 +31,7 @@ %%% it will be stopped after end_per_suite and so on. See terminate %%% documentation for a table describing the scoping machanics. %%% -%%% All of callbacks except init/1 in a CTH are optional.</p> +%%% All of callbacks except init/1 in a CTH are optional. -module(verify_config). @@ -69,14 +69,14 @@ -record(state, { id = ?MODULE :: term()}). -%% @doc Always called before any other callback function. Use this to initiate +%% Always called before any other callback function. Use this to initiate %% any common state. It should return an state for this CTH. -spec init(Id :: term(), Opts :: proplists:proplist()) -> {ok, State :: #state{}}. init(Id, Opts) -> {ok,Opts}. -%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's +%% The ID is used to uniquly identify an CTH instance, if two CTH's %% return the same ID the seconds CTH is ignored. This function should NOT %% have any side effects as it might be called multiple times by common test. -spec id(Opts :: proplists:proplist()) -> @@ -84,7 +84,7 @@ init(Id, Opts) -> id(Opts) -> os:timestamp(). -%% @doc Called before init_per_suite is called. Note that this callback is +%% Called before init_per_suite is called. Note that this callback is %% only called if the CTH is added before init_per_suite is run (eg. in a test %% specification, suite/0 function etc). %% You can change the config in the this function. @@ -95,7 +95,7 @@ id(Opts) -> pre_init_per_suite(Suite,Config,State) -> {Config, State}. -%% @doc Called after init_per_suite. +%% Called after init_per_suite. %% you can change the return value in this function. -spec post_init_per_suite(Suite :: atom(), Config :: config(), @@ -105,7 +105,7 @@ pre_init_per_suite(Suite,Config,State) -> post_init_per_suite(Suite,Config,Return,State) -> {Return, State}. -%% @doc Called before end_per_suite. The config/state can be changed here, +%% Called before end_per_suite. The config/state can be changed here, %% though it will only affect the *end_per_suite function. -spec pre_end_per_suite(Suite :: atom(), Config :: config() | skip_or_fail(), @@ -114,7 +114,7 @@ post_init_per_suite(Suite,Config,Return,State) -> pre_end_per_suite(Suite,Config,State) -> {Config, State}. -%% @doc Called after end_per_suite. Note that the config cannot be +%% Called after end_per_suite. Note that the config cannot be %% changed here, only the status of the suite. -spec post_end_per_suite(Suite :: atom(), Config :: config(), @@ -124,7 +124,7 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> {Return, State}. -%% @doc Called before each init_per_group. +%% Called before each init_per_group. %% You can change the config in this function. -spec pre_init_per_group(Group :: atom(), Config :: config(), @@ -133,7 +133,7 @@ post_end_per_suite(Suite,Config,Return,State) -> pre_init_per_group(Group,Config,State) -> {Config, State}. -%% @doc Called after each init_per_group. +%% Called after each init_per_group. %% You can change the return value in this function. -spec post_init_per_group(Group :: atom(), Config :: config(), @@ -143,7 +143,7 @@ pre_init_per_group(Group,Config,State) -> post_init_per_group(Group,Config,Return,State) -> {Return, State}. -%% @doc Called after each end_per_group. The config/state can be changed here, +%% Called after each end_per_group. The config/state can be changed here, %% though it will only affect the *end_per_group functions. -spec pre_end_per_group(Group :: atom(), Config :: config() | skip_or_fail(), @@ -152,7 +152,7 @@ post_init_per_group(Group,Config,Return,State) -> pre_end_per_group(Group,Config,State) -> {Config, State}. -%% @doc Called after each end_per_group. Note that the config cannot be +%% Called after each end_per_group. Note that the config cannot be %% changed here, only the status of the group. -spec post_end_per_group(Group :: atom(), Config :: config(), @@ -162,7 +162,7 @@ pre_end_per_group(Group,Config,State) -> post_end_per_group(Group,Config,Return,State) -> {Return, State}. -%% @doc Called before each test case. +%% Called before each test case. %% You can change the config in this function. -spec pre_init_per_testcase(TC :: atom(), Config :: config(), @@ -171,7 +171,7 @@ post_end_per_group(Group,Config,Return,State) -> pre_init_per_testcase(TC,Config,State) -> {Config, State}. -%% @doc Called after each test case. Note that the config cannot be +%% Called after each test case. Note that the config cannot be %% changed here, only the status of the test case. -spec post_end_per_testcase(TC :: atom(), Config :: config(), @@ -198,7 +198,7 @@ post_end_per_testcase(TC,Config,Return,State) -> end, {Return, State}. -%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_tc if the suite, group or test case failed. %% This function should be used for extra cleanup which might be needed. %% It is not possible to modify the config or the status of the test run. @@ -209,7 +209,7 @@ post_end_per_testcase(TC,Config,Return,State) -> on_tc_fail(TC, Reason, State) -> State. -%% @doc Called when a test case is skipped by either user action +%% Called when a test case is skipped by either user action %% or due to an init function failing. Test case can be %% end_per_suite, init_per_group, end_per_group and the actual test cases. -spec on_tc_skip(TC :: end_per_suite | @@ -221,7 +221,7 @@ on_tc_fail(TC, Reason, State) -> on_tc_skip(TC, Reason, State) -> State. -%% @doc Called when the scope of the CTH is done, this depends on +%% Called when the scope of the CTH is done, this depends on %% when the CTH was specified. This translation table describes when this %% function is called. %% diff --git a/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl b/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl index 07b21b4178..e4c7be3cac 100644 --- a/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl +++ b/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ %% %CopyrightEnd% %% -%%% @doc Event handler module +%%% Event handler module %%% -%%% <p>This is an event handler module used for testing that -%%% Common Test generates events as expected.</p> +%%% This is an event handler module used for testing that +%%% Common Test generates events as expected. %%% -module(eh_A). diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 8ba14e63bc..340b8f3d52 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -73,18 +73,22 @@ all() -> all(suite) -> lists:reverse( [ + crash_groups, crash_all, bad_return_groups, bad_return_all, + illegal_values_groups, illegal_values_all, alter_groups, alter_all, + alter_all_to_skip, alter_all_from_skip, one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, faulty_cth_exit_in_init, faulty_cth_exit_in_id, faulty_cth_exit_in_init_scope_suite, minimal_cth, minimal_and_maximal_cth, faulty_cth_undef, scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, + scope_suite_group_only_cth, scope_per_suite_state_cth, scope_per_group_state_cth, scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, skip_pre_init_tc_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, - state_update_cth, options_cth, same_id_cth, + state_update_cth, update_result_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, no_init_suite_config, no_init_config, no_end_config, failed_sequence, repeat_force_stop, config_clash, @@ -152,6 +156,11 @@ scope_suite_cth(Config) when is_list(Config) -> do_test(scope_suite_cth, "ct_scope_suite_cth_SUITE.erl", [],Config). +scope_suite_group_only_cth(Config) when is_list(Config) -> + do_test(scope_suite_group_only_cth, + "ct_scope_suite_group_only_cth_SUITE.erl", + [],Config,ok,2,[{group,g1}]). + scope_per_group_cth(Config) when is_list(Config) -> do_test(scope_per_group_cth, "ct_scope_per_group_cth_SUITE.erl", [],Config). @@ -209,6 +218,10 @@ state_update_cth(Config) when is_list(Config) -> do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl", [state_update_cth,state_update_cth],Config). +update_result_cth(Config) -> + do_test(update_result_cth, "ct_cth_update_result_post_end_tc_SUITE.erl", + [update_result_post_end_tc_cth],Config). + options_cth(Config) when is_list(Config) -> do_test(options_cth, "ct_cth_empty_SUITE.erl", [{empty_cth,[test]}],Config). @@ -257,16 +270,21 @@ cth_log(Config) when is_list(Config) -> lists:foreach( fun(UnexpIoLog) -> {ok,Bin} = file:read_file(UnexpIoLog), - Ts = string:tokens(binary_to_list(Bin),[$\n]), - Matches = lists:foldl(fun([$=,$E,$R,$R,$O,$R|_], N) -> - N+1; - ([$L,$o,$g,$g,$e,$r|_], N) -> - N+1; + Ts = string:lexemes(binary_to_list(Bin),[$\n]), + Matches = lists:foldl(fun([$=,$E,$R,$R,$O,$R|_], {E,I,L}) -> + {E+1,I,L}; + ([$=,$I,$N,$F,$O|_], {E,I,L}) -> + {E,I+1,L}; + ([$L,$o,$g,$g,$e,$r|_], {E,I,L}) -> + {E,I,L+1}; (_, N) -> N - end, 0, Ts), - ct:pal("~p matches in ~tp", [Matches,UnexpIoLog]), - if Matches > 10 -> ok; - true -> exit({no_unexpected_io_found,UnexpIoLog}) + end, {0,0,0}, Ts), + ct:pal("~p ({Error,Info,Log}) matches in ~tp", + [Matches,UnexpIoLog]), + MatchList = tuple_to_list(Matches), + case [N || N <- MatchList, N<3] of + [] -> ok; + _ -> exit({missing_unexpected_io,UnexpIoLog}) end end, UnexpIoLogs), ok. @@ -295,10 +313,74 @@ repeat_force_stop(Config) -> [{force_stop,skip_rest},{duration,"000009"}]). %% Test that expected callbacks, and only those, are called when a test -%% are fails due to clash in config alias names +%% fails due to clash in config alias names config_clash(Config) -> do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config). +%% Test post_groups and post_all hook callbacks, introduced by OTP-14746 +alter_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_groups_return,[{new_group,[tc1,tc2]}]}, + {post_all_return,[{group,new_group}]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,[tc2]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all_from_skip(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{all_return,{skip,"skipped by all/0"}}, + {post_all_return,[tc2]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all_to_skip(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_all_return,{skip,"skipped by post_all/3"}}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +bad_return_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,not_a_list}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +bad_return_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,not_a_list}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +illegal_values_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_groups_return,[{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +illegal_values_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_all_return,[{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}]}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +crash_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,crash}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +crash_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,crash}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -318,6 +400,7 @@ do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) -> filename:join([DataDir,"cth/tests",Wildcard])), {Opts,ERPid} = setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config), + Res = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -343,6 +426,13 @@ reformat(Events, EH) -> %reformat(Events, _EH) -> % Events. +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- @@ -361,13 +451,16 @@ test_events(one_empty_cth) -> {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{empty_cth,id,[[]]}}, {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + %% check that post_groups and post_all comes after init when hook + %% is installed with start flag/option. + {?eh,cth,{empty_cth,post_groups,[ct_cth_empty_SUITE,[]]}}, + {?eh,cth,{empty_cth,post_all,[ct_cth_empty_SUITE,[test_case],[]]}}, {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{empty_cth,pre_init_per_suite, [ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_suite, [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, @@ -576,6 +669,10 @@ test_events(scope_suite_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + %% check that post_groups and post_all comes before init when hook + %% is installed in suite/0 + {?eh,cth,{'_',post_groups,['_',[]]}}, + {?eh,cth,{'_',post_all,['_','_',[]]}}, {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, @@ -597,6 +694,34 @@ test_events(scope_suite_cth) -> {?eh,stop_logging,[]} ]; +test_events(scope_suite_group_only_cth) -> + Suite = ct_scope_suite_group_only_cth_SUITE, + CTH = empty_cth, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,1}}, + %% check that post_groups and post_all comes before init when hook + %% is installed in suite/0 + {?eh,cth,{CTH,post_groups,['_',['_']]}}, + {negative, + {?eh,cth,{CTH,post_all,['_','_','_']}}, + {?eh,tc_start,{Suite,init_per_suite}}}, + {?eh,cth,{CTH,id,[[]]}}, + {?eh,cth,{CTH,init,['_',[]]}}, + {?eh,cth,{CTH,pre_init_per_suite,[Suite,'$proplist',mystate]}}, + {?eh,cth,{CTH,post_init_per_suite,[Suite,'$proplist','$proplist',mystate]}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,cth,{CTH,pre_end_per_suite,[Suite,'$proplist',mystate]}}, + {?eh,cth,{CTH,post_end_per_suite,[Suite,'$proplist','_',mystate]}}, + {?eh,cth,{CTH,terminate,[mystate]}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(scope_per_group_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -656,6 +781,8 @@ test_events(scope_suite_state_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{'_',post_groups,['_',[]]}}, + {?eh,cth,{'_',post_all,['_','_',[]]}}, {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,init_per_suite}}, {?eh,cth,{'_',id,[[test]]}}, {?eh,cth,{'_',init,['_',[test]]}}, @@ -1094,6 +1221,106 @@ test_events(state_update_cth) -> {?eh,stop_logging,[]} ]; +test_events(update_result_cth) -> + Suite = ct_cth_update_result_post_end_tc_SUITE, + [ + {?eh,start_logging,'_'}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,tc_start,{Suite,init_per_suite}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,tc_ok_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_fail,'_',ok,[]]}}, + {?eh,tc_done,{Suite,tc_ok_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{0,1,{0,0}}}, + + {?eh,tc_start,{Suite,tc_ok_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_skip,'_',ok,[]]}}, + {?eh,tc_done,{Suite,tc_ok_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{0,1,{1,0}}}, + + {?eh,tc_start,{Suite,tc_fail_to_ok}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_fail_to_ok,'_', + {error,{test_case_failed,"should be changed to ok"}},[]]}}, + {?eh,tc_done,{Suite,tc_fail_to_ok,ok}}, + {?eh,test_stats,{1,1,{1,0}}}, + + {?eh,tc_start,{Suite,tc_fail_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_fail_to_skip,'_', + {error,{test_case_failed,"should be changed to skip"}},[]]}}, + {?eh,tc_done,{Suite,tc_fail_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{1,1,{2,0}}}, + + {?eh,tc_start,{Suite,tc_timetrap_to_ok}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_timetrap_to_ok,'_',{timetrap_timeout,3000},[]]}}, + {?eh,tc_done,{Suite,tc_timetrap_to_ok,ok}}, + {?eh,test_stats,{2,1,{2,0}}}, + + {?eh,tc_start,{Suite,tc_timetrap_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_timetrap_to_skip,'_',{timetrap_timeout,3000},[]]}}, + {?eh,tc_done,{Suite,tc_timetrap_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{2,1,{3,0}}}, + + {?eh,tc_start,{Suite,tc_skip_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_skip_to_fail,'_', + {skip,"should be changed to fail"},[]]}}, + {?eh,tc_done,{Suite,tc_skip_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{2,2,{3,0}}}, + + {?eh,tc_start,{Suite,end_fail_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_fail_to_fail,'_', + {failed, + {Suite,end_per_testcase, + {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}}, + {?eh,tc_done,{Suite,end_fail_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{2,3,{3,0}}}, + + {?eh,tc_start,{Suite,end_fail_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_fail_to_skip,'_', + {failed, + {Suite,end_per_testcase, + {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}}, + {?eh,tc_done,{Suite,end_fail_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{2,3,{4,0}}}, + + {?eh,tc_start,{Suite,end_timetrap_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_timetrap_to_fail,'_', + {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}}, + {?eh,tc_done,{Suite,end_timetrap_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{2,4,{4,0}}}, + + {?eh,tc_start,{Suite,end_timetrap_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_timetrap_to_skip,'_', + {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}}, + {?eh,tc_done,{Suite,end_timetrap_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{2,4,{5,0}}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(options_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -2204,6 +2431,229 @@ test_events(config_clash) -> %% Make sure no 'cth_error' events are received! [{negative,{?eh,cth_error,'_'},E} || E <- Events]; +test_events(alter_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{new_group,[tc1,tc2]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}], + [{new_group,[tc1,tc2]}]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{new_group,[tc1,tc2]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}], + [{new_group,[tc1,tc2]}]]}}, + {?eh,tc_start,{all_and_groups_SUITE,{init_per_group,new_group,[]}}}, + {?eh,tc_done,{all_and_groups_SUITE, + {init_per_group,new_group,'$proplist'},ok}}, + {?eh,tc_start,{all_and_groups_SUITE,tc1}}, + {?eh,tc_done,{all_and_groups_SUITE,tc1,ok}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,tc_start,{all_and_groups_SUITE,{end_per_group,new_group,[]}}}, + {?eh,tc_done,{all_and_groups_SUITE, + {end_per_group,new_group,'$proplist'},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2], + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all_from_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2], + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all_to_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE, + {skip,"skipped by post_all/3"}, + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE, + {skip,"skipped by post_all/3"}, + '_']}}, + {?eh,tc_user_skip,{all_and_groups_SUITE,all,"skipped by post_all/3"}}, + {?eh,cth,{'_',on_tc_skip,[all_and_groups_SUITE,all, + {tc_user_skip,"skipped by post_all/3"}, + []]}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(illegal_values_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups, + [all_and_groups_SUITE, + [{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups, + [all_and_groups_SUITE, + [{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite,{failed,{error,'_'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(illegal_values_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all, + [all_and_groups_SUITE, + [{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}],'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all, + [all_and_groups_SUITE, + [{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}],'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error,'Invalid reference to group this_group_does_not_exist in all_and_groups_SUITE:all/0'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(bad_return_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error, + {'Bad return value from post_groups/2 hook function',not_a_list}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(bad_return_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error,{'Bad return value from post_all/3 hook function',not_a_list}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(crash_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed, + {error,"all_and_groups_cth:post_groups/2 CTH call failed"}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(crash_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed, + {error,"all_and_groups_cth:post_all/3 CTH call failed"}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(ok) -> ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl new file mode 100644 index 0000000000..adc86005f9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl @@ -0,0 +1,47 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(all_and_groups_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_group(_Group,Config) -> + Config. + +end_per_group(_Group,Config) -> + ok. + +all() -> + ct:get_config(all_return,[{group,test_group}]). + +groups() -> + [{test_group,[tc1]}]. + +%% Test cases starts here. +tc1(Config) -> + ok. + +tc2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl new file mode 100644 index 0000000000..9ebc00e9de --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl @@ -0,0 +1,100 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(all_and_groups_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments +-define(fail(Info), + gen_event:notify( + ?CT_EVMGR_REF, + #event{ name = cth_error, + node = node(), + data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})). + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +post_groups(Suite,Groups) -> + case empty_cth:post_groups(Suite,ct:get_config(post_groups_return,Groups)) of + crash -> error(crash_in_post_groups); + R -> R + end. + +post_all(Suite,Tests,Groups) -> + case empty_cth:post_all(Suite,ct:get_config(post_all_return,Tests),Groups) of + crash -> error(crash_in_post_all); + R -> R + end. + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC,Reason,State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC,Reason,State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl new file mode 100644 index 0000000000..a16138ce6f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl @@ -0,0 +1,101 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_cth_update_result_post_end_tc_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + [{timetrap,{seconds,3}}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(EndTimetrap,_) when EndTimetrap==end_timetrap_to_fail; + EndTimetrap==end_timetrap_to_skip-> + timer:sleep(10000); +end_per_testcase(EndFail,_) when EndFail==end_fail_to_fail; + EndFail==end_fail_to_skip-> + ct:fail("change result when end fails"); +end_per_testcase(_,_) -> + ok. + +all() -> + [tc_ok_to_fail, + tc_ok_to_skip, + tc_fail_to_ok, + tc_fail_to_skip, + tc_timetrap_to_ok, + tc_timetrap_to_skip, + tc_skip_to_fail, + end_fail_to_fail, + end_fail_to_skip, + end_timetrap_to_fail, + end_timetrap_to_skip]. + +%% Test cases starts here. +tc_ok_to_fail(_Config) -> + ok. + +tc_ok_to_skip(_Config) -> + ok. + +tc_fail_to_ok(_Config) -> + ct:fail("should be changed to ok"). + +tc_fail_to_skip(_Config) -> + ct:fail("should be changed to skip"). + +tc_timetrap_to_ok(_Config) -> + timer:sleep(10000), % will time out after 3 sek + ok. + +tc_timetrap_to_skip(_Config) -> + timer:sleep(10000), % will time out after 3 sek + ok. + +tc_skip_to_fail(_Config) -> + {skip,"should be changed to fail"}. + +end_fail_to_fail(_Config) -> + ok. + +end_fail_to_skip(_Config) -> + ok. + +end_timetrap_to_fail(_Config) -> + ok. + +end_timetrap_to_skip(_Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl new file mode 100644 index 0000000000..38c9da903d --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(ct_match_state_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +post_groups(Suite, Groups) -> + empty_cth:post_groups(Suite, Groups). + +post_all(Suite, Tests, Groups) -> + empty_cth:post_all(Suite, Tests, Groups). + +init(Id, Opts) -> + empty_cth:init(Id, Opts), + {ok,mystate}. + +%% In the following, always match against the state value, to ensure +%% that init has indeed been called before the rest of the hooks. +pre_init_per_suite(Suite,Config,mystate) -> + empty_cth:pre_init_per_suite(Suite,Config,mystate). + +post_init_per_suite(Suite,Config,Return,mystate) -> + empty_cth:post_init_per_suite(Suite,Config,Return,mystate). + +pre_end_per_suite(Suite,Config,mystate) -> + empty_cth:pre_end_per_suite(Suite,Config,mystate). + +post_end_per_suite(Suite,Config,Return,mystate) -> + empty_cth:post_end_per_suite(Suite,Config,Return,mystate). + +terminate(mystate) -> + empty_cth:terminate(mystate). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl index d400348354..404fb33476 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_per_tc_cth_SUITE.erl @@ -1,7 +1,7 @@ %%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2018. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -29,7 +29,6 @@ %% Test server callback functions
%%--------------------------------------------------------------------
-%% @doc
%% Config - [tuple()]
%% A list of key/value pairs, holding the test case configuration.
%%
@@ -38,27 +37,23 @@ %% Note: This function is free to add any key/value pairs to the Config
%% variable, but should NOT alter/remove any existing entries.
%%
-%% @spec init_per_suite(Config) -> Config
-%% @end
+%% -spec init_per_suite(Config) -> Config
%%--------------------------------------------------------------------
init_per_suite(Config) ->
Config.
%%--------------------------------------------------------------------
-%% @doc
%% Config - [tuple()]
%% A list of key/value pairs, holding the test case configuration.
%%
%% Cleanup after the whole suite
%%
-%% @spec end_per_suite(Config) -> _
-%% @end
+%% -spec end_per_suite(Config) -> _
%%--------------------------------------------------------------------
end_per_suite(_Config) ->
ok.
%%--------------------------------------------------------------------
-%% @doc
%% Case - atom()
%% Name of the test case that is about to be run.
%% Config - [tuple()]
@@ -70,14 +65,12 @@ end_per_suite(_Config) -> %% variable, but should NOT alter/remove any existing entries.
%% Initiation before each test case
%%
-%% @spec init_per_testcase(TestCase, Config) -> Config
-%% @end
+%% -spec init_per_testcase(TestCase, Config) -> Config
%%--------------------------------------------------------------------
init_per_testcase(_TestCase, Config) ->
[{ct_hooks,[empty_cth]}|Config].
%%--------------------------------------------------------------------
-%% @doc
%% Case - atom()
%% Name of the test case that is about to be run.
%% Config - [tuple()]
@@ -85,22 +78,19 @@ init_per_testcase(_TestCase, Config) -> %%
%% Cleanup after each test case
%%
-%% @spec end_per_testcase(TestCase, Config) -> _
-%% @end
+%% -spec end_per_testcase(TestCase, Config) -> _
%%--------------------------------------------------------------------
end_per_testcase(_TestCase, _Config) ->
ok.
%%--------------------------------------------------------------------
-%% @doc
%% TestCases - [Case]
%% Case - atom()
%% Name of a test case.
%%
%% Returns a list of all test cases in this test suite
%%
-%% @spec all() -> TestCases
-%% @end
+%% -spec all() -> TestCases
%%--------------------------------------------------------------------
all() ->
[test_case].
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl new file mode 100644 index 0000000000..537c97d3f0 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl @@ -0,0 +1,54 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_scope_suite_group_only_cth_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +suite() -> + [{ct_hooks,[ct_match_state_cth]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [test_case]. + +groups() -> + [{g1,[test_case]}]. + +%% Test cases starts here. +test_case(Config) when is_list(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl index bd1ac54781..eda190b682 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/cth_log_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -124,6 +124,6 @@ gen() -> gen_loop(N) -> ct:log("Logger iteration: ~p", [N]), error_logger:error_report(N), - error_logger:info_report(progress, N), + error_logger:info_report(N), ct:sleep(150), gen_loop(N+1). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl index 961ea68d2d..60488e84c6 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,9 +18,9 @@ %% %CopyrightEnd% %% -%%% @doc Common Test Example Suite Callback module. +%%% Common Test Example Suite Callback module. %%% -%%% <p>This module gives an example of a common test CTH (Common Test Hook). +%%% This module gives an example of a common test CTH (Common Test Hook). %%% There are many ways to add a CTH to a test run, you can do it either in %%% the command line using -ct_hook, in a test spec using %%% {ct_hook,M} or in the suite it self by returning ct_hook @@ -31,7 +31,7 @@ %%% it will be stopped after end_per_suite and so on. See terminate %%% documentation for a table describing the scoping machanics. %%% -%%% All of callbacks except init/1 in a CTH are optional.</p> +%%% All of callbacks except init/1 in a CTH are optional. -module(empty_cth). @@ -39,6 +39,9 @@ -export([id/1]). -export([init/2]). +-export([post_all/3]). +-export([post_groups/2]). + -export([pre_init_per_suite/3]). -export([post_init_per_suite/4]). -export([pre_end_per_suite/3]). @@ -71,7 +74,32 @@ -record(state, { id = ?MODULE :: term()}). -%% @doc Always called before any other callback function. Use this to initiate +%% Called after groups/0. +%% You can change the return value in this function. +-spec post_groups(Suite :: atom(), Groups :: list()) -> list(). +post_groups(Suite,Groups) -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_groups, + [Suite,Groups]}}), + ct:log("~w:post_groups(~w) called", [?MODULE,Suite]), + Groups. + +%% Called after all/0. +%% You can change the return value in this function. +-spec post_all(Suite :: atom(), + Tests :: list(), + Groups :: term()) -> + list(). +post_all(Suite,Tests,Groups) -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_all, + [Suite,Tests,Groups]}}), + ct:log("~w:post_all(~w) called", [?MODULE,Suite]), + Tests. + +%% Always called before any other callback function. Use this to initiate %% any common state. It should return an state for this CTH. -spec init(Id :: term(), Opts :: proplists:proplist()) -> {ok, State :: #state{}}. @@ -81,7 +109,7 @@ init(Id, Opts) -> ct:log("~w:init called", [?MODULE]), {ok,Opts}. -%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's +%% The ID is used to uniquly identify an CTH instance, if two CTH's %% return the same ID the seconds CTH is ignored. This function should NOT %% have any side effects as it might be called multiple times by common test. -spec id(Opts :: proplists:proplist()) -> @@ -92,7 +120,7 @@ id(Opts) -> ct:log("~w:id called", [?MODULE]), ct_test_support:unique_timestamp(). -%% @doc Called before init_per_suite is called. Note that this callback is +%% Called before init_per_suite is called. Note that this callback is %% only called if the CTH is added before init_per_suite is run (eg. in a test %% specification, suite/0 function etc). %% You can change the config in the this function. @@ -108,7 +136,7 @@ pre_init_per_suite(Suite,Config,State) -> ct:log("~w:pre_init_per_suite(~w) called", [?MODULE,Suite]), {Config, State}. -%% @doc Called after init_per_suite. +%% Called after init_per_suite. %% you can change the return value in this function. -spec post_init_per_suite(Suite :: atom(), Config :: config(), @@ -123,7 +151,7 @@ post_init_per_suite(Suite,Config,Return,State) -> ct:log("~w:post_init_per_suite(~w) called", [?MODULE,Suite]), {Return, State}. -%% @doc Called before end_per_suite. The config/state can be changed here, +%% Called before end_per_suite. The config/state can be changed here, %% though it will only affect the *end_per_suite function. -spec pre_end_per_suite(Suite :: atom(), Config :: config() | skip_or_fail(), @@ -137,7 +165,7 @@ pre_end_per_suite(Suite,Config,State) -> ct:log("~w:pre_end_per_suite(~w) called", [?MODULE,Suite]), {Config, State}. -%% @doc Called after end_per_suite. Note that the config cannot be +%% Called after end_per_suite. Note that the config cannot be %% changed here, only the status of the suite. -spec post_end_per_suite(Suite :: atom(), Config :: config(), @@ -152,7 +180,7 @@ post_end_per_suite(Suite,Config,Return,State) -> ct:log("~w:post_end_per_suite(~w) called", [?MODULE,Suite]), {Return, State}. -%% @doc Called before each init_per_group. +%% Called before each init_per_group. %% You can change the config in this function. -spec pre_init_per_group(Suite :: atom(), Group :: atom(), @@ -167,7 +195,7 @@ pre_init_per_group(Suite,Group,Config,State) -> ct:log("~w:pre_init_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Config, State}. -%% @doc Called after each init_per_group. +%% Called after each init_per_group. %% You can change the return value in this function. -spec post_init_per_group(Suite :: atom(), Group :: atom(), @@ -183,7 +211,7 @@ post_init_per_group(Suite,Group,Config,Return,State) -> ct:log("~w:post_init_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Return, State}. -%% @doc Called after each end_per_group. The config/state can be changed here, +%% Called after each end_per_group. The config/state can be changed here, %% though it will only affect the *end_per_group functions. -spec pre_end_per_group(Suite :: atom(), Group :: atom(), @@ -198,7 +226,7 @@ pre_end_per_group(Suite,Group,Config,State) -> ct:log("~w:pre_end_per_group(~w~w) called", [?MODULE,Suite,Group]), {Config, State}. -%% @doc Called after each end_per_group. Note that the config cannot be +%% Called after each end_per_group. Note that the config cannot be %% changed here, only the status of the group. -spec post_end_per_group(Suite :: atom(), Group :: atom(), @@ -214,7 +242,7 @@ post_end_per_group(Suite,Group,Config,Return,State) -> ct:log("~w:post_end_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Return, State}. -%% @doc Called before init_per_testcase/2 for each test case. +%% Called before init_per_testcase/2 for each test case. %% You can change the config in this function. -spec pre_init_per_testcase(Suite :: atom(), TC :: atom(), @@ -229,7 +257,7 @@ pre_init_per_testcase(Suite,TC,Config,State) -> ct:log("~w:pre_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Config, State}. -%% @doc Called after init_per_testcase/2, and before the test case. +%% Called after init_per_testcase/2, and before the test case. -spec post_init_per_testcase(Suite :: atom(), TC :: atom(), Config :: config(), @@ -244,7 +272,7 @@ post_init_per_testcase(Suite,TC,Config,Return,State) -> ct:log("~w:post_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Return, State}. -%% @doc Called before end_per_testacse/2. No skip or fail allowed here, +%% Called before end_per_testacse/2. No skip or fail allowed here, %% only config additions. -spec pre_end_per_testcase(Suite :: atom(), TC :: atom(), @@ -259,7 +287,7 @@ pre_end_per_testcase(Suite,TC,Config,State) -> ct:log("~w:pre_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Config, State}. -%% @doc Called after end_per_testcase/2 for each test case. Note that +%% Called after end_per_testcase/2 for each test case. Note that %% the config cannot be changed here, only the status of the test case. -spec post_end_per_testcase(Suite :: atom(), TC :: atom(), @@ -275,7 +303,7 @@ post_end_per_testcase(Suite,TC,Config,Return,State) -> ct:log("~w:post_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Return, State}. -%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, +%% Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_tc if the suite, group or test case failed. %% This function should be used for extra cleanup which might be needed. %% It is not possible to modify the config or the status of the test run. @@ -292,7 +320,7 @@ on_tc_fail(Suite, TC, Reason, State) -> ct:log("~w:on_tc_fail(~w,~w) called", [?MODULE,Suite,TC]), State. -%% @doc Called when a test case is skipped by either user action +%% Called when a test case is skipped by either user action %% or due to an init function failing. Test case can be %% end_per_suite, init_per_group, end_per_group and the actual test cases. -spec on_tc_skip(Suite :: atom(), @@ -310,7 +338,7 @@ on_tc_skip(Suite, TC, Reason, State) -> ct:log("~w:on_tc_skip(~w,~w) called", [?MODULE,Suite,TC]), State. -%% @doc Called when the scope of the CTH is done, this depends on +%% Called when the scope of the CTH is done, this depends on %% when the CTH was specified. This translation table describes when this %% function is called. %% diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl new file mode 100644 index 0000000000..7afb3d8781 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl @@ -0,0 +1,98 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(update_result_post_end_tc_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State), + change_result(TC,Config,State). + +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). + +%%%----------------------------------------------------------------- +%%% +change_result(tc_ok_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(tc_ok_to_skip,_Config,State) -> + {{skip, "Test skipped"}, State}; +change_result(tc_fail_to_ok,Config,State) -> + {lists:keydelete(tc_status,1,Config),State}; +change_result(tc_fail_to_skip,Config,State) -> + {{skip,"Test skipped"},State}; +change_result(tc_timetrap_to_ok,Config,State) -> + {lists:keydelete(tc_status,1,Config),State}; +change_result(tc_timetrap_to_skip,Config,State) -> + {{skip,"Test skipped"},State}; +change_result(tc_skip_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(end_fail_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(end_fail_to_skip,_Config,State) -> + {{skip, "Test skipped"}, State}; +change_result(end_timetrap_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(end_timetrap_to_skip,_Config,State) -> + {{skip, "Test skipped"}, State}. diff --git a/lib/common_test/test/ct_log_SUITE.erl b/lib/common_test/test/ct_log_SUITE.erl index 93affda398..214eb17e1a 100644 --- a/lib/common_test/test/ct_log_SUITE.erl +++ b/lib/common_test/test/ct_log_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -174,7 +174,7 @@ verify(Config) -> TcLogFile = proplists:get_value(the_logfile, SavedCfg), Pid = proplists:get_value(the_pid, SavedCfg), StrPid = lists:flatten(io_lib:format("~p",[Pid])), - EscPid = "<" ++ string:substr(StrPid, 2, length(StrPid)-2) ++ ">", + EscPid = "<" ++ string:slice(StrPid, 1, length(StrPid)-2) ++ ">", String = proplists:get_value(the_string, SavedCfg), ct:log("Read from prev testcase: ~p & ~p", [TcLogFile,Pid]), {ok,Dev} = file:open(TcLogFile, [read]), diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 586589ad40..0d17481e95 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2017. All Rights Reserved. +%% Copyright Ericsson AB 2013-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,9 +23,7 @@ %% Description: %% This file contains the test cases for the ct_netconfc API. %% -%% @author Support -%% @doc Netconf Client Interface. -%% @end +%% Netconf Client Interface. %%---------------------------------------------------------------------- %%---------------------------------------------------------------------- -module(netconfc1_SUITE). @@ -442,6 +440,12 @@ edit_config(Config) -> ?ok = ct_netconfc:edit_config(Client,running, {server,[{xmlns,"myns"}], [{name,["myserver"]}]}), + ?NS:expect_reply('edit-config',ok), + ?ok = ct_netconfc:edit_config(Client,running, + [{server,[{xmlns,"myns"}], + [{name,["server1"]}]}, + {server,[{xmlns,"myns"}], + [{name,["server2"]}]}]), ?NS:expect_do_reply('close-session',close,ok), ?ok = ct_netconfc:close_session(Client), ok. diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 63bf9be134..656fdb4a40 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2017. All Rights Reserved. +%% Copyright Ericsson AB 2012-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -231,8 +231,7 @@ data_for_channel(CM, Ch, Data, State) -> {ok, NewState} end catch - Class:Reason -> - Stacktrace = erlang:get_stacktrace(), + Class:Reason:Stacktrace -> error_logger:error_report([{?MODULE, data_for_channel}, {request, Data}, {buffer, State#session.buffer}, diff --git a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl index 7ffe6f045b..bca2d5f3de 100644 --- a/lib/common_test/test/ct_pre_post_test_io_SUITE.erl +++ b/lib/common_test/test/ct_pre_post_test_io_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -164,7 +164,7 @@ pre_post_io(Config) -> fun(PrePostIoFile) -> ct:log("Reading Pre/Post Test IO Log file: ~ts", [PrePostIoFile]), {ok,Bin} = file:read_file(PrePostIoFile), - Ts = string:tokens(binary_to_list(Bin),[$\n]), + Ts = string:lexemes(binary_to_list(Bin),[$\n]), PrePostIOEntries = lists:foldl(fun([$L,$o,$g,$g,$e,$r|_], {pre,PreLogN,PreErrN,0,0}) -> @@ -203,7 +203,7 @@ pre_post_io(Config) -> fun(UnexpIoFile) -> ct:log("Reading Unexpected IO Log file: ~ts", [UnexpIoFile]), {ok,Bin} = file:read_file(UnexpIoFile), - Ts = string:tokens(binary_to_list(Bin),[$\n]), + Ts = string:lexemes(binary_to_list(Bin),[$\n]), UnexpIOEntries = lists:foldl(fun([$L,$o,$g,$g,$e,$r|_], [LogN,ErrN]) -> [LogN+1,ErrN]; @@ -241,7 +241,7 @@ try_loop(_Fun, 0) -> gave_up; try_loop(Fun, N) -> try Fun() of - {error,_} -> + {Error,_} when Error==error; Error==badrpc -> timer:sleep(10), try_loop(Fun, N-1); Result -> @@ -257,7 +257,7 @@ try_loop(M, F, _A, 0) -> gave_up; try_loop(M, F, A, N) -> try apply(M, F, A) of - {error,_} -> + {Error,_Reason} when Error==error; Error==badrpc -> timer:sleep(10), try_loop(M, F, A, N-1); Result -> diff --git a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl index 1b171801a3..0e10ec187d 100644 --- a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl +++ b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -126,12 +126,12 @@ default(Config) -> auto_per_tc(Config) -> PrivDir = proplists:get_value(priv_dir, Config), - ["log_private",_] = string:tokens(filename:basename(PrivDir), "."), + ["log_private",_] = string:lexemes(filename:basename(PrivDir), "."), {ok,_} = file:list_dir(PrivDir). manual_per_tc(Config) -> PrivDir = proplists:get_value(priv_dir, Config), - ["log_private",_] = string:tokens(filename:basename(PrivDir), "."), + ["log_private",_] = string:lexemes(filename:basename(PrivDir), "."), {error,_} = file:list_dir(PrivDir), ok = ct:make_priv_dir(), {ok,_} = file:list_dir(PrivDir). diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl index 4ce375b4ee..1a305b1516 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE_data/a_test/r1_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,14 +19,6 @@ %% %%---------------------------------------------------------------------- %% File: r1_SUITE.erl -%% -%% Description: -%% -%% -%% @author Support -%% @doc -%% @end -%%---------------------------------------------------------------------- %%---------------------------------------------------------------------- -module(r1_SUITE). -include_lib("common_test/include/ct.hrl"). diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl index 77bb544080..393cbcc780 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE_data/b_test/r2_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -19,14 +19,6 @@ %% %%---------------------------------------------------------------------- %% File: r2_SUITE.erl -%% -%% Description: -%% -%% -%% @author Support -%% @doc -%% @end -%%---------------------------------------------------------------------- %%---------------------------------------------------------------------- -module(r2_SUITE). -include_lib("common_test/include/ct.hrl"). diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl index a6533641d8..1c81bbe95d 100644 --- a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl +++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,9 +23,7 @@ %% Description: %% This file contains the test cases for the ct_snmp API. %% -%% @author Support -%% @doc Test of SNMP support in common_test -%% @end +%% Test of SNMP support in common_test %%---------------------------------------------------------------------- %%---------------------------------------------------------------------- -module(snmp_SUITE). diff --git a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl index ed10356cdd..7a73e45b43 100644 --- a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,9 +23,7 @@ %% Description: %% This file contains the test cases for cth_surefire. %% -%% @author Support -%% @doc Test of surefire support in common_test -%% @end +%% Test of surefire support in common_test %%---------------------------------------------------------------------- %%---------------------------------------------------------------------- -module(surefire_SUITE). diff --git a/lib/common_test/test/ct_tc_repeat_SUITE.erl b/lib/common_test/test/ct_tc_repeat_SUITE.erl new file mode 100644 index 0000000000..433b5456fe --- /dev/null +++ b/lib/common_test/test/ct_tc_repeat_SUITE.erl @@ -0,0 +1,438 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_tc_repeat_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{minutes,1}}]. + +all() -> + all(suite). + +all(suite) -> + [ + repeat, + repeat_parallel_until_ok, + repeat_parallel_until_fail, + repeat_sequence_until_ok, + repeat_sequence_until_fail, + pick_one_test_from_group, + pick_one_test_from_subgroup + ]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +%% Test post_groups and post_all hook callbacks, introduced by OTP-14746 +repeat(Config) -> + ok = do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [], [], Config). + +repeat_parallel_until_ok(Config) -> + {error,{{illegal_combination,{parallel,repeat_until_ok}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_ok}], + [], Config, 1, []). + +repeat_parallel_until_fail(Config) -> + {error,{{illegal_combination,{parallel,repeat_until_fail}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_fail}], + [], Config, 1, []). + +repeat_sequence_until_ok(Config) -> + {error,{{illegal_combination,{sequence,repeat_until_ok}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_ok}], + [], Config, 1, []). + +repeat_sequence_until_fail(Config) -> + {error,{{illegal_combination,{sequence,repeat_until_fail}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_fail}], + [], Config, 1, []). + +pick_one_test_from_group(Config) -> + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_mixed},{testcase,tc2}], + [], Config, 1, []). + +pick_one_test_from_subgroup(Config) -> + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", + [{group,[[g_mixed,subgroup]]},{testcase,tc2}], + [], Config, 1, []). + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, Suite, WTT, CTHs, Config) -> + do_test(Tag, Suite, WTT, CTHs, Config, 2, []). + +do_test(Tag, Suite0, WTT, CTHs, Config, EC, ExtraOpts) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join([DataDir,Suite0]), + {Opts,ERPid} = + setup([{suite,Suite}|WTT]++[{ct_hooks,CTHs},{label,Tag}|ExtraOpts], + Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + %% io:format("~p~n",[Events]), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + Res. + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(repeat) -> + S = tc_repeat_SUITE, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,unknown}}, + + %% tc1, {repeat,2} + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + %% tc2, {repeat_until_ok,3} + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + %% tc3, {repeat_until_ok,3} + {?eh,tc_start,{S,tc3}}, + {?eh,tc_done,{tc_repeat_SUITE,tc3, + {failed,{error,{test_case_failed,always_fail}}}}}, + {?eh,test_stats,{3,1,{0,0}}}, + {?eh,tc_start,{S,tc3}}, + {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}}, + {?eh,test_stats,{3,2,{0,0}}}, + {?eh,tc_start,{S,tc3}}, + {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}}, + {?eh,test_stats,{3,3,{0,0}}}, + %% tc4, {repeat_until_fail,3} + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,ok}}, + {?eh,test_stats,{4,3,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, + {?eh,test_stats,{4,4,{0,0}}}, + %% g, tc1, {repeat,2} + {?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{5,4,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{6,4,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}, + %% g_until_ok, tc2, {repeat_until_ok,3} + {?eh,tc_start,{S,{init_per_group,g_until_ok,[]}}}, + {?eh,tc_done,{S,{init_per_group,g_until_ok,[]},ok}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{7,4,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g_until_ok,[]}}}, + {?eh,tc_done,{S,{end_per_group,g_until_ok,[]},ok}}, + %% g_until_fail, tc4, {repeat_until_fail,3} + {?eh,tc_start,{S,{init_per_group,g_until_fail,[]}}}, + {?eh,tc_done,{S,{init_per_group,g_until_fail,[]},ok}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,ok}}, + {?eh,test_stats,{8,4,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, + {?eh,test_stats,{8,5,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g_until_fail,[]}}}, + {?eh,tc_done,{S,{end_per_group,g_until_fail,[]},ok}}, + %% g, parallel, tc1, {repeat,2} + {parallel, + [{?eh,tc_start,{S,{init_per_group,g,[parallel]}}}, + {?eh,tc_done,{S,{init_per_group,g,[parallel]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{9,5,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{10,5,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[parallel]}}}, + {?eh,tc_done,{S,{end_per_group,g,[parallel]},ok}}]}, + %% g, sequence, tc1, {repeat,2} + {?eh,tc_start,{S,{init_per_group,g,[sequence]}}}, + {?eh,tc_done,{S,{init_per_group,g,[sequence]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{11,5,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{12,5,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[sequence]}}}, + {?eh,tc_done,{S,{end_per_group,g,[sequence]},ok}}, + %% g_sequence_skip_rest, + {?eh,tc_start,{S,{init_per_group,g_mixed,[sequence]}}}, + {?eh,tc_done,{S,{init_per_group,g_mixed,[sequence]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{13,5,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{14,5,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,ok}}, + {?eh,test_stats,{15,5,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, + {?eh,test_stats,{15,6,{0,0}}}, + %% ----> fail in sequence, so skip rest + {?eh,tc_auto_skip,{S,{tc4,g_mixed}, % last of current repeat tc4 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,1}}}, + {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,2}}}, + {?eh,tc_auto_skip,{S,{tc1,g}, % group g, tc1, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,3}}}, + {?eh,tc_auto_skip,{S,{tc1,subgroup}, % subgroup, single tc1 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,4}}}, + {?eh,tc_auto_skip,{S,{tc2,subgroup}, % subgroup, tc2, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,5}}}, + {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % tc2, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,6}}}, + {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % single tc2 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,7}}}, + {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % tc1, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,8}}}, + {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,9}}}, + {?eh,tc_start,{S,{end_per_group,g_mixed,'_'}}}, + {?eh,tc_done,{S,{end_per_group,g_mixed,'_'},ok}}, + %% done + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_parallel_until_ok) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_parallel_until_ok,[parallel]}}}, + {?eh,tc_done,{'_',{init_per_group,g_parallel_until_ok,[parallel]},ok}}, + {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_ok}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_parallel_until_fail) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_parallel_until_fail,[parallel]}}}, + {?eh,tc_done,{'_',{init_per_group,g_parallel_until_fail,[parallel]},ok}}, + {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_fail}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_sequence_until_ok) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_sequence_until_ok,[sequence]}}}, + {?eh,tc_done,{'_',{init_per_group,g_sequence_until_ok,[sequence]},ok}}, + {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_ok}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_sequence_until_fail) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_sequence_until_fail,[sequence]}}}, + {?eh,tc_done,{'_',{init_per_group,g_sequence_until_fail,[sequence]},ok}}, + {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_fail}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(pick_one_test_from_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}}, + {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',tc2}}}, % single tc2 + {?eh,tc_done,{'_',tc2,ok}}, + {?eh,tc_start,{'_',tc2}}, % tc2, {repeat,2} + {?eh,tc_done,{'_',tc2,ok}}, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_done,{'_',tc2,ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}}, + {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(pick_one_test_from_subgroup) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}}, + {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_start,{'_',{init_per_group,subgroup,[]}}}}, + {?eh,tc_done,{'_',{init_per_group,subgroup,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',tc2}}}, % tc2, {repeat,2} + {?eh,tc_done,{'_',tc2,ok}}, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_done,{'_',tc2,ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',{end_per_group,subgroup,[]}}}}, + {?eh,tc_done,{'_',{end_per_group,subgroup,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}}, + {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(ok) -> + ok. + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl b/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl new file mode 100644 index 0000000000..f5d960d12f --- /dev/null +++ b/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl @@ -0,0 +1,85 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(tc_repeat_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_group(_Group,Config) -> + Config. + +end_per_group(_Group,Config) -> + ok. + +all() -> + [{testcase,tc1,[{repeat,2}]}, + {testcase,tc2,[{repeat_until_ok,3}]}, + {testcase,tc3,[{repeat_until_ok,3}]}, + {testcase,tc4,[{repeat_until_fail,3}]}, + {group,g}, + {group,g_until_ok}, + {group,g_until_fail}, + {group,g,[parallel]}, + {group,g,[sequence]}, + {group,g_mixed,[sequence]}]. + +groups() -> + [{g,[{testcase,tc1,[{repeat,2}]}]}, + {g_until_ok,[{testcase,tc2,[{repeat_until_ok,3}]}]}, + {g_until_fail,[{testcase,tc4,[{repeat_until_fail,3}]}]}, + {g_parallel_until_ok,[parallel],[{testcase,tc2,[{repeat_until_ok,3}]}]}, + {g_parallel_until_fail,[parallel],[{testcase,tc1,[{repeat_until_fail,2}]}]}, + {g_sequence_until_ok,[sequence],[{testcase,tc2,[{repeat_until_ok,3}]}]}, + {g_sequence_until_fail,[sequence],[{testcase,tc1,[{repeat_until_fail,2}]}]}, + {g_mixed,[{testcase,tc1,[{repeat,2}]}, + {testcase,tc4,[{repeat,3}]}, + tc1, + {group,g}, + {subgroup,[tc1,{testcase,tc2,[{repeat,2}]}]}, + {testcase,tc2,[{repeat,2}]}, + tc2, + {testcase,tc1,[{repeat,2}]}, + tc1]}]. + +%% Test cases starts here. +tc1(_Config) -> + ok. + +tc2(_Config) -> + ok. + +tc3(_Config) -> + ct:fail(always_fail). + +tc4(Config) -> + case ?config(saved_config,Config) of + {tc4,_} -> + ct:fail(second_time_fail); + undefined -> + {save_config,Config} + end. + +tc5(_Config) -> + {skip,"just skip this"}. diff --git a/lib/common_test/test/ct_telnet_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE.erl index a0089c9bc9..f71b7c370f 100644 --- a/lib/common_test/test/ct_telnet_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE.erl @@ -50,10 +50,10 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. groups() -> - [{legacy, [], [unix_telnet,own_server,timetrap]}, - {raw, [], [unix_telnet,own_server,timetrap]}, - {html, [], [unix_telnet,own_server]}, - {silent, [], [unix_telnet,own_server]}]. + [{legacy, [], [unix_telnet,own_server,faulty_regexp,timetrap]}, + {raw, [], [unix_telnet,own_server,faulty_regexp,timetrap]}, + {html, [], [unix_telnet,own_server,faulty_regexp]}, + {silent, [], [unix_telnet,own_server,faulty_regexp]}]. all() -> [ @@ -119,6 +119,12 @@ own_server(Config) -> all_tests_in_suite(own_server,"ct_telnet_own_server_SUITE", CfgFile,Config). +faulty_regexp(Config) -> + CfgFile = "telnet.faulty_regexp." ++ + atom_to_list(groupname(Config)) ++ ".cfg", + all_tests_in_suite(faulty_regexp,"ct_telnet_faulty_regexp_SUITE", + CfgFile,Config). + timetrap(Config) -> CfgFile = "telnet.timetrap." ++ atom_to_list(groupname(Config)) ++ ".cfg", @@ -225,6 +231,31 @@ events_to_check(unix_telnet,Config) -> all_cases(ct_telnet_basic_SUITE,Config); events_to_check(own_server,Config) -> all_cases(ct_telnet_own_server_SUITE,Config); +events_to_check(faulty_regexp,_Config) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_pattern, + {failed, + {error,{{bad_pattern,"invalid(pattern",{"missing )",15}}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_pattern_no_string, + {failed, + {error,{{bad_pattern,invalid_pattern}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_tag_pattern, + {failed, + {error,{{bad_pattern,{tag,"invalid(pattern"},{"missing )",15}}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_tag_pattern_no_string, + {failed, + {error,{{bad_pattern,{tag,invalid_pattern}}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done,{ct_telnet_faulty_regexp_SUITE,expect_pattern_unicode,ok}}, + {?eh,tc_done,{ct_telnet_faulty_regexp_SUITE,expect_tag_pattern_unicode,ok}}, + {?eh,stop_logging,[]}]; events_to_check(timetrap,_Config) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_done,{ct_telnet_timetrap_SUITE,expect_timetrap, diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_faulty_regexp_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_faulty_regexp_SUITE.erl new file mode 100644 index 0000000000..a5c9451a9c --- /dev/null +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_faulty_regexp_SUITE.erl @@ -0,0 +1,79 @@ +-module(ct_telnet_faulty_regexp_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(name, telnet_server_conn1). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +suite() -> [{require,?name,{unix,[telnet]}}, + {require,ct_conn_log}, + {ct_hooks, [{cth_conn_log,[]}]}]. + +all() -> + [expect_pattern, + expect_pattern_no_string, + expect_tag_pattern, + expect_tag_pattern_no_string, + expect_pattern_unicode, + expect_tag_pattern_unicode]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +init_per_testcase(_,Config) -> + ct:log("init_per_testcase: opening telnet connection...",[]), + {ok,_} = ct_telnet:open(?name), + ct:log("...done",[]), + Config. + +end_per_testcase(_,_Config) -> + ct:log("end_per_testcase: closing telnet connection...",[]), + _ = ct_telnet:close(?name), + ct:log("...done",[]), + ok. + +expect_pattern(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, "invalid(pattern"). + +expect_pattern_no_string(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, invalid_pattern). + +expect_tag_pattern(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, {tag,"invalid(pattern"}). + +expect_tag_pattern_no_string(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, {tag,invalid_pattern}). + +%% Test that a unicode pattern can be given without the testcase +%% failing. Do however notice that there is no real unicode support +%% in ct_telnet yet, that is, the telnet binary mode is not supported. +expect_pattern_unicode(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + {error,{prompt,_}} = ct_telnet:expect(?name, "pattern_with_unicode_αβ"), + ok. + +expect_tag_pattern_unicode(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + {error,{prompt,_}} = ct_telnet:expect(?name, "pattern_with_unicode_αβ"), + ok. diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl index 985fa40ad2..34df57027e 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl @@ -58,7 +58,8 @@ all() -> server_speaks, server_disconnects, newline_ayt, - newline_break + newline_break, + newline_string ]. groups() -> @@ -393,3 +394,11 @@ newline_break(_) -> "> " = lists:flatten(R), ok = ct_telnet:close(Handle), ok. + +%% Test option {newline,String} to specify an own newline, e.g. "\r\n" +newline_string(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo hello-", [{newline,"own_nl\n"}]), + {ok,["hello-own_nl"]} = ct_telnet:expect(Handle, ["hello-own_nl"]), + ok = ct_telnet:close(Handle), + ok. diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 44c27e54c2..388d5d46c6 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2017. All Rights Reserved. +%% Copyright Ericsson AB 2008-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,9 +18,9 @@ %% %CopyrightEnd% %% -%%% @doc Test support functions +%%% Test support functions %%% -%%% <p>This is a support module for testing the Common Test Framework.</p> +%%% This is a support module for testing the Common Test Framework. %%% -module(ct_test_support). @@ -88,7 +88,7 @@ start_slave(Config, Level) -> start_slave(ct, Config, Level). start_slave(NodeName, Config, Level) -> - [_,Host] = string:tokens(atom_to_list(node()), "@"), + [_,Host] = string:lexemes(atom_to_list(node()), "@"), test_server:format(0, "Trying to start ~s~n", [atom_to_list(NodeName)++"@"++Host]), PR = proplists:get_value(printable_range,Config,io:printable_range()), @@ -1088,8 +1088,8 @@ locate({TEH,Name,{'DEF','RUNDIR'}}, Node, [Ev|Evs], Config) -> {TEH,#event{name=Name, node=Node, data=EvData}} -> {_,{_,LogDir}} = lists:keysearch(logdir, 1, get_opts(Config)), D = filename:join(LogDir, "ct_run." ++ atom_to_list(Node)), - case string:str(EvData, D) of - 0 -> exit({badmatch,EvData}); + case string:find(EvData, D) of + nomatch -> exit({badmatch,EvData}); _ -> ok end, {Config,Evs}; @@ -1104,8 +1104,8 @@ locate({TEH,Name,{'DEF',{'START_TIME','LOGDIR'}}}, Node, [Ev|Evs], Config) -> {DT={{_,_,_},{_,_,_}},Dir} when is_list(Dir) -> {_,{_,LogDir}} = lists:keysearch(logdir, 1, get_opts(Config)), D = filename:join(LogDir, "ct_run." ++ atom_to_list(Node)), - case string:str(Dir, D) of - 0 -> exit({badmatch,Dir}); + case string:find(Dir, D) of + nomatch -> exit({badmatch,Dir}); _ -> ok end, {[{start_time,DT}|Config],Evs}; @@ -1373,7 +1373,7 @@ delete_dirs(LogDir) -> Dirs2Del = lists:foldl(fun(Dir, Del) -> [S,Mi,H,D,Mo,Y|_] = - lists:reverse(string:tokens(Dir, [$.,$-,$_])), + lists:reverse(string:lexemes(Dir, [$.,$-,$_])), S2I = fun(Str) -> list_to_integer(Str) end, DT = {{S2I(Y),S2I(Mo),S2I(D)}, {S2I(H),S2I(Mi),S2I(S)}}, Then = calendar:datetime_to_gregorian_seconds(DT), diff --git a/lib/common_test/test/ct_test_support_eh.erl b/lib/common_test/test/ct_test_support_eh.erl index e8db52dcd3..120692dc7c 100644 --- a/lib/common_test/test/ct_test_support_eh.erl +++ b/lib/common_test/test/ct_test_support_eh.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,10 +18,10 @@ %% %CopyrightEnd% %% -%%% @doc Event handler module +%%% Event handler module %%% -%%% <p>This is an event handler module used for testing that -%%% Common Test generates events as expected.</p> +%%% This is an event handler module used for testing that +%%% Common Test generates events as expected. %%% -module(ct_test_support_eh). diff --git a/lib/common_test/test/ct_unicode_SUITE.erl b/lib/common_test/test/ct_unicode_SUITE.erl index 355503a5dc..6f6ec97ceb 100644 --- a/lib/common_test/test/ct_unicode_SUITE.erl +++ b/lib/common_test/test/ct_unicode_SUITE.erl @@ -191,7 +191,7 @@ check_logs(Dirs) -> [] -> ok; Match -> - MatchStr = string:join(Match,"\n"), + MatchStr = lists:join("\n",Match), ct:log("ERROR: Escaped unicode characters found in:~n~ts",[MatchStr]), ct:fail(escaped_unicode_characters_found) end. diff --git a/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl b/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl index 993452500e..4f9dd20e3e 100644 --- a/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl +++ b/lib/common_test/test/ct_unicode_SUITE_data/unicode_atoms_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% Copyright Ericsson AB 2009-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -77,7 +77,7 @@ all() -> 'fail_αβ_4'(_Config) -> ct:log("This is test case ~tw",[?FUNCTION_NAME]), - S = try throw(ok) catch throw:ok -> erlang:get_stacktrace() end, + S = try throw(ok) catch throw:ok:Stacktrace -> Stacktrace end, erlang:raise(error,{error,testcase,?FUNCTION_NAME},S), ok. diff --git a/lib/common_test/test/ct_userconfig_callback.erl b/lib/common_test/test/ct_userconfig_callback.erl index c723f4ca1c..e48d338dd5 100644 --- a/lib/common_test/test/ct_userconfig_callback.erl +++ b/lib/common_test/test/ct_userconfig_callback.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ -export([check_parameter/1, read_config/1]). read_config(Str) -> - KeyVals = string:tokens(Str, " "), + KeyVals = string:lexemes(Str, " "), {ok,read_config1(KeyVals)}. read_config1([Key,Val | KeyVals]) -> diff --git a/lib/common_test/test/ct_util_SUITE.erl b/lib/common_test/test/ct_util_SUITE.erl new file mode 100644 index 0000000000..1d773855da --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE.erl @@ -0,0 +1,490 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_util_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + CTHs = filelib:wildcard(filename:join(DataDir,"*_cth.erl")), + io:format("CTHs: ~p",[CTHs]), + [io:format("Compiling ~p: ~p", + [FileName,compile:file(FileName,[{outdir,DataDir},debug_info])]) || + FileName <- CTHs], + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{minutes,1}}]. + +all() -> + all(suite). + +all(suite) -> + [ + pre_init_per_suite, + post_init_per_suite, + pre_end_per_suite, + post_end_per_suite, + pre_init_per_group, + post_init_per_group, + pre_end_per_group, + post_end_per_group, + pre_init_per_testcase, + post_init_per_testcase, + pre_end_per_testcase, + post_end_per_testcase + ]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +pre_init_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + + +pre_init_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_init_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_testcase, + {curr_tc_SUITE,tc1,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_testcase, + {curr_tc_SUITE,tc1,{timeout,5000}}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_testcase, + {curr_tc_SUITE,tc1,{timeout,5000}}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_testcase, + {curr_tc_SUITE,tc1,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, Suite, RunTestArgs, Config) -> + do_test(Tag, Suite, RunTestArgs, Config, 2). + +do_test(Tag, Suite0, RunTestArgs, Config, EC) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join([DataDir,Suite0]), + {Opts,ERPid} = setup([{suite,Suite}]++[{label,Tag}|RunTestArgs],Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + %% io:format("~p~n",[Events]), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + Res. + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(IPS) when IPS=:=pre_init_per_suite; IPS=:=post_init_per_suite -> + S = curr_tc_SUITE, + FwFunc = + case IPS of + pre_init_per_suite -> init_tc; + post_init_per_suite -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,E}}, + {?eh,tc_auto_skip,{S,tc1,{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,tc2,{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_suite,E}}}}, + {?eh,test_stats,{0,0,{0,4}}}, + {?eh,tc_auto_skip,{S,end_per_suite,{failed,{S,init_per_suite,E}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPS) when EPS=:=pre_end_per_suite; EPS=:=post_end_per_suite -> + S = curr_tc_SUITE, + FwFunc = + case EPS of + pre_end_per_suite -> init_tc; + post_end_per_suite -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,E}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(IPG) when IPG=:=pre_init_per_group; IPG=:=post_init_per_group -> + S = curr_tc_SUITE, + FwFunc = + case IPG of + pre_init_per_group -> init_tc; + post_init_per_group -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},E}}, + {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_group,E}}}}, + {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_group,E}}}}, + {?eh,test_stats,{2,0,{0,2}}}, + {?eh,tc_auto_skip,{S,{end_per_group,g},{failed,{S,init_per_group,E}}}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPG) when EPG=:=pre_end_per_group; EPG=:=post_end_per_group -> + S = curr_tc_SUITE, + FwFunc = + case EPG of + pre_end_per_group -> init_tc; + post_end_per_group -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},E}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(IPTC) when IPTC=:=pre_init_per_testcase; + IPTC=:=post_init_per_testcase -> + S = curr_tc_SUITE, + E = case IPTC of + pre_init_per_testcase -> + {failed,{ct_framework,init_tc,{test_case_failed,hahahahahah}}}; + post_init_per_testcase -> + {failed,{ct_framework,end_tc,{timetrap,3000}}} + end, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,{auto_skipped,E}}}, + {?eh,test_stats,{0,0,{0,1}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{1,0,{0,1}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,{auto_skipped,E}}}, + {?eh,test_stats,{1,0,{0,2}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,2}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPTC) when EPTC=:=pre_end_per_testcase; EPTC=:=post_end_per_testcase-> + S = curr_tc_SUITE, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl b/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl new file mode 100644 index 0000000000..34c1568a87 --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl @@ -0,0 +1,105 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(ct_util_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + erlang:system_time(second). + +init(Id, Opts) -> + {ok,ok}. + +pre_init_per_suite(Suite,Config,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Config, State}. + +post_init_per_suite(Suite,Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Return, State}. + +pre_end_per_suite(Suite,Config,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Config, State}. + +post_end_per_suite(Suite,Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Return, State}. + +pre_init_per_group(Suite, Group, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Config,State}. + +post_init_per_group(Suite, Group, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Return,State}. + +pre_end_per_group(Suite, Group, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Config,State}. + +post_end_per_group(Suite, Group, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Return,State}. + +pre_init_per_testcase(Suite, TC, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Config,State}. + +post_init_per_testcase(Suite, TC, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Return,State}. + +pre_end_per_testcase(Suite, TC, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Config,State}. + +post_end_per_testcase(Suite, TC, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Return,State}. + +%%%----------------------------------------------------------------- +maybe_sleep(FuncName,Suite) -> + maybe_sleep(FuncName,Suite,undefined). +maybe_sleep(FuncName,Suite,GroupOrTC) -> + case ct:get_config(FuncName) of + {Suite,GroupOrTC,Fail} -> + fail(Fail); + {Suite,Fail} when GroupOrTC=:=undefined -> + fail(Fail); + _ -> + ok + end. + +fail({timeout,T}) -> + timer:sleep(T); +fail(kill) -> + spawn_link(fun() -> ct:fail(hahahahahah) end), + timer:sleep(10000). + diff --git a/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl b/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl new file mode 100644 index 0000000000..b48ba4d24e --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl @@ -0,0 +1,59 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(curr_tc_SUITE). + +-suite_defaults([{timetrap, {seconds, 3}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_suite(Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + Config. + +end_per_suite(Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +init_per_group(_Group,Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + Config. + +end_per_group(_Group,Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +all() -> + [tc1,tc2,{group,g}]. + +groups() -> + [{g,[tc1,tc2]}]. + +%% Test cases starts here. +tc1(_Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +tc2(_Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. diff --git a/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl b/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl index 03a0832e53..d1c293b4af 100644 --- a/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl +++ b/lib/common_test/test/ct_verbosity_SUITE_data/simple_evh.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,14 +18,14 @@ %% %CopyrightEnd% %% -%%% @doc Common Test Framework Event Handler +%%% Common Test Framework Event Handler %%% -%%% <p>This module implements an event handler that CT uses to +%%% This module implements an event handler that CT uses to %%% handle status and progress notifications during test runs. %%% The notifications are handled locally (per node) and passed %%% on to ct_master when CT runs in distributed mode. This %%% module may be used as a template for other event handlers -%%% that can be plugged in to handle local logging and reporting.</p> +%%% that can be plugged in to handle local logging and reporting. -module(simple_evh). -behaviour(gen_event). diff --git a/lib/common_test/test/erl2html2_SUITE.erl b/lib/common_test/test/erl2html2_SUITE.erl index 53a63578b2..b2336ff0bc 100644 --- a/lib/common_test/test/erl2html2_SUITE.erl +++ b/lib/common_test/test/erl2html2_SUITE.erl @@ -214,10 +214,10 @@ check_line_number(Last,Line,OrigLine) -> [$>|Rest] = lists:dropwhile(fun($>) -> false; (_) -> true end,Line), check_line_number(Last,Rest,OrigLine); _ -> - [N |_] = string:tokens(Line,":"), + [N |_] = string:lexemes(Line,":"), % erlang:display(N), Num = - try list_to_integer(string:strip(N)) + try list_to_integer(string:trim(N,both,"\s")) catch _:_ -> ct:fail({no_line_number_after,Last,OrigLine}) end, if Num == Last+1 -> diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl index 65300b0bdf..cef7784333 100644 --- a/lib/common_test/test/telnet_server.erl +++ b/lib/common_test/test/telnet_server.erl @@ -249,7 +249,7 @@ do_handle_data("echo " ++ Data,State) -> send(Data++"\r\n> ",State), {ok,State}; do_handle_data("echo_sep " ++ Data,State) -> - Msgs = string:tokens(Data," "), + Msgs = string:lexemes(Data," "), lists:foreach(fun(Msg) -> send(Msg,State), timer:sleep(10) @@ -260,28 +260,28 @@ do_handle_data("echo_no_prompt " ++ Data,State) -> send(Data,State), {ok,State}; do_handle_data("echo_ml " ++ Data,State) -> - Lines = string:tokens(Data," "), - ReturnData = string:join(Lines,"\n"), + Lines = string:lexemes(Data," "), + ReturnData = lists:flatten(lists:join("\n",Lines)), send(ReturnData++"\r\n> ",State), {ok,State}; do_handle_data("echo_ml_no_prompt " ++ Data,State) -> - Lines = string:tokens(Data," "), - ReturnData = string:join(Lines,"\n"), + Lines = string:lexemes(Data," "), + ReturnData = lists:flatten(lists:join("\n",Lines)), send(ReturnData,State), {ok,State}; do_handle_data("echo_loop " ++ Data,State) -> - [TStr|Lines] = string:tokens(Data," "), - ReturnData = string:join(Lines,"\n"), + [TStr|Lines] = string:lexemes(Data," "), + ReturnData = lists:flatten(lists:join("\n",Lines)), send_loop(list_to_integer(TStr),ReturnData,State), {ok,State}; do_handle_data("echo_delayed_prompt "++Data,State) -> - [MsStr|EchoData] = string:tokens(Data, " "), - send(string:join(EchoData,"\n"),State), + [MsStr|EchoData] = string:lexemes(Data, " "), + send(lists:flatten(lists:join("\n",EchoData)),State), timer:sleep(list_to_integer(MsStr)), send("\r\n> ",State), {ok,State}; do_handle_data("disconnect_after " ++WaitStr,State) -> - Wait = list_to_integer(string:strip(WaitStr,right,$\n)), + Wait = list_to_integer(string:trim(WaitStr,trailing,"\n")), dbg("Server will close connection in ~w ms...", [Wait]), erlang:send_after(Wait,self(),disconnect), {ok,State}; diff --git a/lib/common_test/test/test_server_SUITE.erl b/lib/common_test/test/test_server_SUITE.erl index 50d8bdd1ac..05737cfac9 100644 --- a/lib/common_test/test/test_server_SUITE.erl +++ b/lib/common_test/test/test_server_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2018. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,12 +18,7 @@ %% %CopyrightEnd% %% %%%------------------------------------------------------------------- -%%% @author Lukas Larsson <[email protected]> -%%% @copyright (C) 2011, Erlang Solutions Ltd. -%%% @doc -%%% -%%% @end -%%% Created : 15 Feb 2011 by Lukas Larsson <[email protected]> +%%% Author: Lukas Larsson <[email protected]> %%%------------------------------------------------------------------- -module(test_server_SUITE). @@ -71,7 +66,7 @@ init_per_testcase(_TestCase, Config) -> %% @spec end_per_testcase(TestCase, Config0) -> %% void() | {save_config,Config1} | {fail,Reason} end_per_testcase(test_server_unicode, _Config) -> - [_,Host] = string:tokens(atom_to_list(node()), "@"), + [_,Host] = string:lexemes(atom_to_list(node()), "@"), N1 = list_to_atom("test_server_tester_latin1" ++ "@" ++ Host), N2 = list_to_atom("test_server_tester_utf8" ++ "@" ++ Host), test_server:stop_node(N1), @@ -347,7 +342,7 @@ generate_and_run_unicode_test(Config0,Encoding) -> RunDir = get_latest_run_dir(LogDir), true = filelib:is_dir(RunDir), - LowerModStr = string:to_lower(ModStr), + LowerModStr = string:lowercase(ModStr), SuiteHtml = translate_filename(LowerModStr++".src.html",Encoding), true = filelib:is_regular(filename:join(RunDir,SuiteHtml)), @@ -362,7 +357,7 @@ generate_and_run_unicode_test(Config0,Encoding) -> %% remote file system on master - i.e. they will use same file name %% mode as the master. start_node(Config,Name,Args) -> - [_,Host] = string:tokens(atom_to_list(node()), "@"), + [_,Host] = string:lexemes(atom_to_list(node()), "@"), ct:log("Trying to start ~w@~s~n",[Name,Host]), case test_server:start_node(Name, peer, [{args,Args}]) of {error,Reason} -> diff --git a/lib/common_test/test/test_server_test_lib.erl b/lib/common_test/test/test_server_test_lib.erl index c18b89b178..9ee946af0b 100644 --- a/lib/common_test/test/test_server_test_lib.erl +++ b/lib/common_test/test/test_server_test_lib.erl @@ -43,7 +43,7 @@ pre_init_per_testcase(_TC,Config,State) -> {start_slave(Config, 50),State}. start_slave(Config,_Level) -> - [_,Host] = string:tokens(atom_to_list(node()), "@"), + [_,Host] = string:lexemes(atom_to_list(node()), "@"), ct:log("Trying to start ~s~n", ["test_server_tester@"++Host]), @@ -121,7 +121,7 @@ parse_suite(FileName) -> end. fline(Fd) -> - case prim_file:read_line(Fd) of + case file:read_line(Fd) of eof -> eof; {ok, Line} -> Line end. |