From 2a3fc026842241b0a1e1b2b1f691bd212499ffe0 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sun, 15 Jul 2012 01:48:03 +0200 Subject: Implement support for test case execution break/continue --- lib/common_test/src/ct.erl | 87 ++++++++++++++++++-- lib/common_test/src/ct_framework.erl | 23 +++++- lib/common_test/src/ct_logs.erl | 4 +- lib/common_test/src/ct_run.erl | 152 ++++++++++++++++++++++------------- lib/test_server/src/test_server.erl | 55 ++++++++----- 5 files changed, 236 insertions(+), 85 deletions(-) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 571d99029f..4054b441ee 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -66,7 +66,8 @@ capture_start/0, capture_stop/0, capture_get/0, capture_get/1, fail/1, fail/2, comment/1, comment/2, make_priv_dir/0, testcases/2, userdata/2, userdata/3, - timetrap/1, get_timetrap_info/0, sleep/1]). + timetrap/1, get_timetrap_info/0, sleep/1, + break/1, break/2, continue/0, continue/1]). %% New API for manipulating with config handlers -export([add_config/2, remove_config/2]). @@ -151,7 +152,8 @@ run(TestDirs) -> %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | %%% {refresh_logs,LogDir} | {logopts,LogOpts} | {basic_html,Bool} | -%%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} +%%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} | +%%% {noinput,Bool} %%% TestDirs = [string()] | string() %%% Suites = [string()] | [atom()] | string() | atom() %%% Cases = [atom()] | atom() @@ -840,10 +842,11 @@ userdata(TestDir, Suite, Case) when is_atom(Case) -> %%%----------------------------------------------------------------- %%% @spec get_status() -> TestStatus | {error,Reason} | no_tests_running %%% TestStatus = [StatusElem] -%%% StatusElem = {current,{Suite,TestCase}} | {successful,Successful} | +%%% StatusElem = {current,TestCaseInfo} | {successful,Successful} | %%% {failed,Failed} | {skipped,Skipped} | {total,Total} +%%% TestCaseInfo = {Suite,TestCase} | [{Suite,TestCase}] %%% Suite = atom() -%%% TestCase = atom() +%%% TestCase = atom() | %%% Successful = integer() %%% Failed = integer() %%% Skipped = {UserSkipped,AutoSkipped} @@ -853,7 +856,8 @@ userdata(TestDir, Suite, Case) when is_atom(Case) -> %%% Reason = term() %%% %%% @doc Returns status of ongoing test. The returned list contains info about -%%% which test case is currently executing, as well as counters for +%%% which test case is currently executing (a list of cases when a +%%% parallel test case group is executing), as well as counters for %%% successful, failed, skipped, and total test cases so far. get_status() -> case get_testdata(curr_tc) of @@ -878,6 +882,8 @@ get_testdata(Key) -> Error; {'EXIT',_Reason} -> no_tests_running; + [CurrTC] when Key == curr_tc -> + {ok,CurrTC}; Data -> {ok,Data} end. @@ -1047,3 +1053,74 @@ sleep({seconds,Ss}) -> sleep(trunc(Ss * 1000)); sleep(Time) -> test_server:adjusted_sleep(Time). + +%%%----------------------------------------------------------------- +%%% @spec break(Comment) -> ok | {error,Reason} +%%% Comment = string() +%%% Reason = {multiple_cases_running,TestCases} +%%% TestCases = [atom()] +%%% +%%% @doc

This function will cancel all timetraps and pause the +%%% execution of the current test case until the user calls the +%%% continue/0 function. It gives the user the opportunity +%%% to interact with the erlang node running the tests, e.g. for +%%% debugging purposes or for manually executing a part of the +%%% test case. If a parallel group is executing, break/2 +%%% should be called instead.

+break(Comment) -> + case get_testdata(curr_tc) of + {ok,{_,TestCase}} -> + test_server:break(?MODULE, Comment); + {ok,Cases} when is_list(Cases) -> + {error,{multiple_cases_running, + [TC || {_,TC} <- Cases]}}; + Error -> + {error,Error} + end. + +%%%----------------------------------------------------------------- +%%% @spec break(TestCase, Comment) -> ok | {error,Reason} +%%% TestCase = atom() +%%% Comment = string() +%%% Reason = test_case_not_running +%%% +%%% @doc

This function works the same way as break/1, +%%% only the TestCase argument makes it possible to +%%% pause a test case executing in a parallel group. The +%%% continue/1 function should be used to resume +%%% execution of TestCase.

+break(TestCase, Comment) -> + case get_testdata(curr_tc) of + {ok,Cases} when is_list(Cases) -> + case lists:keymember(TestCase, 2, Cases) of + true -> + test_server:break(?MODULE, TestCase, Comment); + false -> + {error,test_case_not_running} + end; + {ok,{_,TestCase}} -> + test_server:break(?MODULE, TestCase, Comment); + Error -> + {error,Error} + end. + +%%%----------------------------------------------------------------- +%%% @spec continue() -> ok +%%% +%%% @doc

This function must be called in order to continue after a +%%% test case (not executing in a parallel group) has called +%%% break/1.

+continue() -> + test_server:continue(). + +%%%----------------------------------------------------------------- +%%% @spec continue(TestCase) -> ok +%%% TestCase = atom() +%%% +%%% @doc

This function must be called in order to continue after a +%%% test case has called break/2. If the paused test case, +%%% TestCase, executes in a parallel group, this +%%% function - rather than continue/0 - must be used +%%% in order to let the test case proceed.

+continue(TestCase) -> + test_server:continue(TestCase). diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 11575cd0fb..0740eb7ec9 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -71,8 +71,13 @@ init_tc(Mod,Func,Config) -> {skip,{require_failed_in_suite0,Reason}}; {Suite,{suite0_failed,_}=Failure} -> {skip,Failure}; - _ -> - ct_util:set_testdata({curr_tc,{Suite,Func}}), + CurrTC -> + case CurrTC of + undefined -> + ct_util:set_testdata({curr_tc,[{Suite,Func}]}); + Running when is_list(Running) -> + ct_util:set_testdata({curr_tc,[{Suite,Func}|Running]}) + end, case ct_util:read_suite_data({seq,Suite,Func}) of undefined -> init_tc1(Mod,Suite,Func,Config); @@ -206,7 +211,8 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) -> case catch configure(MergedInfo,MergedInfo,SuiteInfo, FuncSpec,Config) of {suite0_failed,Reason} -> - ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}), + ct_util:set_testdata({curr_tc,{Mod,{suite0_failed, + {require,Reason}}}}), {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; @@ -633,7 +639,16 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> end, ct_util:reset_silent_connections(), - + + %% reset the curr_tc state, or delete this TC from the list of + %% executing cases (if in a parallel group) + case ct_util:get_testdata(curr_tc) of + Running = [_,_|_] -> + ct_util:set_testdata({curr_tc,lists:delete({Mod,Func}, Running)}); + [_] -> + ct_util:set_testdata({curr_tc,undefined}) + end, + case FinalResult of {skip,{sequence_failed,_,_}} -> %% ct_logs:init_tc is never called for a skipped test case diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 1ccbdc3718..73a66a8763 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -378,11 +378,11 @@ tc_print(Category,Format,Args) -> ok. get_heading(default) -> - io_lib:format("-----------------------------" + io_lib:format("\n-----------------------------" "-----------------------\n~s\n", [log_timestamp(now())]); get_heading(Category) -> - io_lib:format("-----------------------------" + io_lib:format("\n-----------------------------" "-----------------------\n~s ~w\n", [log_timestamp(now()),Category]). diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 46aec04ec1..9b9c43a917 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1428,67 +1428,111 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> "run ct:start_interactive()\n\n",[]), {error,interactive_mode}; _Pid -> - %% save stylesheet info - ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), - %% save logopts - ct_util:set_testdata({logopts,Opts#opts.logopts}), - %% enable silent connections - case Opts#opts.silent_connections of - [] -> - Conns = ct_util:override_silence_all_connections(), - ct_logs:log("Silent connections", "~p", [Conns]); - Conns when is_list(Conns) -> - ct_util:override_silence_connections(Conns), - ct_logs:log("Silent connections", "~p", [Conns]); - _ -> - ok - end, - log_ts_names(Opts1#opts.testspecs), - TestSuites = suite_tuples(Tests), - - {_TestSuites1,SuiteMakeErrors,AllMakeErrors} = - case application:get_env(common_test, auto_compile) of - {ok,false} -> - {TestSuites1,SuitesNotFound} = - verify_suites(TestSuites), - {TestSuites1,SuitesNotFound,SuitesNotFound}; - _ -> - {SuiteErrs,HelpErrs} = auto_compile(TestSuites), - {TestSuites,SuiteErrs,SuiteErrs++HelpErrs} - end, + compile_and_run(Tests, Skip, Opts1, Args) + end + end. - case continue(AllMakeErrors) of - true -> - SavedErrors = save_make_errors(SuiteMakeErrors), - ct_repeat:log_loop_info(Args), +compile_and_run(Tests, Skip, Opts, Args) -> + %% save stylesheet info + ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), + %% save logopts + ct_util:set_testdata({logopts,Opts#opts.logopts}), + %% enable silent connections + case Opts#opts.silent_connections of + [] -> + Conns = ct_util:override_silence_all_connections(), + ct_logs:log("Silent connections", "~p", [Conns]); + Conns when is_list(Conns) -> + ct_util:override_silence_connections(Conns), + ct_logs:log("Silent connections", "~p", [Conns]); + _ -> + ok + end, + log_ts_names(Opts#opts.testspecs), + TestSuites = suite_tuples(Tests), + + {_TestSuites1,SuiteMakeErrors,AllMakeErrors} = + case application:get_env(common_test, auto_compile) of + {ok,false} -> + {TestSuites1,SuitesNotFound} = + verify_suites(TestSuites), + {TestSuites1,SuitesNotFound,SuitesNotFound}; + _ -> + {SuiteErrs,HelpErrs} = auto_compile(TestSuites), + {TestSuites,SuiteErrs,SuiteErrs++HelpErrs} + end, + + case continue(AllMakeErrors) of + true -> + SavedErrors = save_make_errors(SuiteMakeErrors), + ct_repeat:log_loop_info(Args), + + {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors), + + possibly_spawn(true == proplists:get_value(noinput, Args), + Tests1, Skip1, Opts); + false -> + io:nl(), + ct_util:stop(clean), + BadMods = + lists:foldr( + fun({{_,_},Ms}, Acc) -> + Ms ++ lists:foldl( + fun(M, Acc1) -> + lists:delete(M, Acc1) + end, Acc, Ms) + end, [], AllMakeErrors), + {error,{make_failed,BadMods}} + end. - {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors), +%% keep the shell as the top controlling process +possibly_spawn(false, Tests, Skip, Opts) -> + TestResult = (catch do_run_test(Tests, Skip, Opts)), + case TestResult of + {EType,_} = Error when EType == user_error; + EType == error -> + ct_util:stop(clean), + exit(Error); + _ -> + ct_util:stop(normal), + TestResult + end; - R = (catch do_run_test(Tests1, Skip1, Opts1)), - case R of - {EType,_} = Error when EType == user_error ; +%% we must return control to the shell now, so we spawn +%% a test supervisor process to keep an eye on the test run +possibly_spawn(true, Tests, Skip, Opts) -> + CTUtilSrv = whereis(ct_util_server), + Supervisor = + fun() -> + process_flag(trap_exit, true), + link(CTUtilSrv), + TestRun = + fun() -> + TestResult = (catch do_run_test(Tests, Skip, Opts)), + case TestResult of + {EType,_} = Error when EType == user_error; EType == error -> ct_util:stop(clean), exit(Error); _ -> ct_util:stop(normal), - R - end; - false -> - io:nl(), - ct_util:stop(clean), - BadMods = - lists:foldr( - fun({{_,_},Ms}, Acc) -> - Ms ++ lists:foldl( - fun(M, Acc1) -> - lists:delete(M, Acc1) - end, Acc, Ms) - end, [], AllMakeErrors), - {error,{make_failed,BadMods}} - end - end - end. + exit({ok,TestResult}) + end + end, + TestRunPid = spawn_link(TestRun), + receive + {'EXIT',TestRunPid,{ok,TestResult}} -> + io:format(user, "~nCommon Test returned ~p~n~n", + [TestResult]); + {'EXIT',TestRunPid,Error} -> + exit(Error) + end + end, + unlink(CTUtilSrv), + SupPid = spawn(Supervisor), + io:format(user, "~nTest control handed over to process ~p~n~n", + [SupPid]), + SupPid. %% attempt to compile the modules specified in TestSuites auto_compile(TestSuites) -> @@ -1900,7 +1944,7 @@ do_run_test(Tests, Skip, Opts) -> [code:del_path(Dir) || Dir <- AddedToPath], ok; Error -> - Error + exit(Error) end. delete_dups([S | Suites]) -> diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 6e94e4861a..44ae89c9b0 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -49,7 +49,7 @@ -export([run_on_shielded_node/2]). -export([is_cover/0,is_debug/0,is_commercial/0]). --export([break/1,continue/0]). +-export([break/1,break/2,break/3,continue/0,continue/1]). %%% DEBUGGER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([purify_new_leaks/0, purify_format/2, purify_new_fds_inuse/0, @@ -2061,31 +2061,40 @@ fail() -> %% Break a test case so part of the test can be done manually. %% Use continue/0 to continue. break(Comment) -> - case erase(test_server_timetraps) of - undefined -> ok; - List -> lists:foreach(fun({Ref,_,_}) -> - timetrap_cancel(Ref) - end, List) - end, + break(?MODULE, Comment). + +break(CBM, Comment) -> + break(CBM, '', Comment). + +break(CBM, TestCase, Comment) -> + timetrap_cancel(), + {TCName,CntArg,PName} = + if TestCase == '' -> + {"", "", test_server_break_process}; + true -> + Str = atom_to_list(TestCase), + {[32 | Str], Str, + list_to_atom("test_server_break_process_" ++ Str)} + end, io:format(user, "\n\n\n--- SEMIAUTOMATIC TESTING ---" - "\nThe test case executes on process ~w" + "\nThe test case~s executes on process ~w" "\n\n\n~s" "\n\n\n-----------------------------\n\n" - "Continue with --> test_server:continue().\n", - [self(),Comment]), - case whereis(test_server_break_process) of + "Continue with --> ~w:continue(~s).\n", + [TCName,self(),Comment,CBM,CntArg]), + case whereis(PName) of undefined -> - spawn_break_process(self()); + spawn_break_process(self(), PName); OldBreakProcess -> OldBreakProcess ! cancel, - spawn_break_process(self()) + spawn_break_process(self(), PName) end, receive continue -> ok end. -spawn_break_process(Pid) -> +spawn_break_process(Pid, PName) -> spawn(fun() -> - register(test_server_break_process,self()), + register(PName, self()), receive continue -> continue(Pid); cancel -> ok @@ -2094,13 +2103,19 @@ spawn_break_process(Pid) -> continue() -> case whereis(test_server_break_process) of - undefined -> - ok; - BreakProcess -> - BreakProcess ! continue + undefined -> ok; + BreakProcess -> BreakProcess ! continue end. -continue(Pid) -> +continue(TestCase) when is_atom(TestCase) -> + PName = list_to_atom("test_server_break_process_" ++ + atom_to_list(TestCase)), + case whereis(PName) of + undefined -> ok; + BreakProcess -> BreakProcess ! continue + end; + +continue(Pid) when is_pid(Pid) -> Pid ! continue. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -- cgit v1.2.3 From c41fbd49054693d61f9f78ef92da176aeea3bf83 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 21 Aug 2012 16:11:42 +0200 Subject: Fix error in documentation --- lib/common_test/src/ct.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 4054b441ee..fe99222927 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -846,7 +846,7 @@ userdata(TestDir, Suite, Case) when is_atom(Case) -> %%% {failed,Failed} | {skipped,Skipped} | {total,Total} %%% TestCaseInfo = {Suite,TestCase} | [{Suite,TestCase}] %%% Suite = atom() -%%% TestCase = atom() | +%%% TestCase = atom() %%% Successful = integer() %%% Failed = integer() %%% Skipped = {UserSkipped,AutoSkipped} -- cgit v1.2.3