aboutsummaryrefslogtreecommitdiffstats
path: root/lib/test_server
diff options
context:
space:
mode:
authorLukas Larsson <[email protected]>2011-02-17 17:07:05 +0100
committerLukas Larsson <[email protected]>2011-02-17 17:09:38 +0100
commit53974a3517268389ae18fabee509ebf4bbfce747 (patch)
tree208575bf9ac4916c6b60e052c6f712986c3f0de9 /lib/test_server
parentb83388e78e72e66f4e0dae42083a6bcd9709922e (diff)
parentd87708bf18d29c0a66f6654368b4553067e62394 (diff)
downloadotp-53974a3517268389ae18fabee509ebf4bbfce747.tar.gz
otp-53974a3517268389ae18fabee509ebf4bbfce747.tar.bz2
otp-53974a3517268389ae18fabee509ebf4bbfce747.zip
Merge branch 'lukas/common_test/suite_callback/OTP-8851' into dev
* lukas/common_test/suite_callback/OTP-8851: (54 commits) Update example cth spec to reflect the implementation Cleanup code to fix dialyzer warning Rename Suite Callback to Common Test hook in documentation Rename Suite Callback to Common Test Hook in code and testcases Fix bug where the state of an SCB was altered when no on_tc_* existed Update SCBs to use a specific id/1 function for SCb overriding instead of returning it from init/1. Add documentation for SCBs Update suite callback test timeout so that beam debug test runs do not timeout Update test support so that if common test fails to be stopped in a suite, the ct node is restarted and the test case fails Add tests for SCB's with same id and fixed bug relating to it Add test suites for failing in init/1 function Added tests for starting SCB's with arguments and fixed a bug with how SCB's are parsed from the command line Add so that failures in SCB:init/1 causes the entire scb scope to fail Fix bug which caused scb to be added too much when initiated in the suite info function Started work on documenting suite callbacks, this is a partial commit Add locking mechanism for scb state when using parallel groups Add state update tests for suite callbacks Update ct_framework calls to allow manipulation of test results in end_tc without breaking backwards compatability (I hope) Add test case for init_per_suite recover scenario Add testcase for config updates ...
Diffstat (limited to 'lib/test_server')
-rw-r--r--lib/test_server/src/test_server.erl132
-rw-r--r--lib/test_server/src/test_server_ctrl.erl2
2 files changed, 77 insertions, 57 deletions
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index ee121e5bb6..13ff02cc6c 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -935,8 +935,7 @@ spawn_fw_call(Mod,{init_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why,
Skip = {skip,{failed,{Mod,init_per_testcase,Why}}},
%% if init_per_testcase fails, the test case
%% should be skipped
- case catch test_server_sup:framework_call(
- end_tc,[?pl2a(Mod),Func,{Pid,Skip,[[]]}]) of
+ case catch do_end_tc_call(Mod,Func,{Pid,Skip,[[]]},Why) of
{'EXIT',FwEndTCErr} ->
exit({fw_notify_done,end_tc,FwEndTCErr});
_ ->
@@ -955,11 +954,9 @@ spawn_fw_call(Mod,{end_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why,
Conf = [{tc_status,ok}],
%% if end_per_testcase fails, the test case should be
%% reported successful with a warning printed as comment
- case catch test_server_sup:framework_call(end_tc,
- [?pl2a(Mod),Func,
- {Pid,
- {failed,{Mod,end_per_testcase,Why}},
- [Conf]}]) of
+ case catch do_end_tc_call(Mod,Func,{Pid,
+ {failed,{Mod,end_per_testcase,Why}},
+ [Conf]}, Why) of
{'EXIT',FwEndTCErr} ->
exit({fw_notify_done,end_tc,FwEndTCErr});
_ ->
@@ -1001,9 +998,7 @@ spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) ->
ok
end,
Conf = [{tc_status,{failed,timetrap_timeout}}],
- case catch test_server_sup:framework_call(end_tc,
- [?pl2a(Mod),Func,
- {Pid,Error,[Conf]}]) of
+ case catch do_end_tc_call(Mod,Func,{Pid,Error,[Conf]},Error) of
{'EXIT',FwEndTCErr} ->
exit({fw_notify_done,end_tc,FwEndTCErr});
_ ->
@@ -1069,27 +1064,27 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
{{Time,Value},Loc,Opts} =
case test_server_sup:framework_call(init_tc,[?pl2a(Mod),Func,Args0],
- {ok,Args0}) of
+ {ok, Args0}) of
{ok,Args} ->
run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback);
Error = {error,_Reason} ->
- test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Error,Args0}]),
- {{0,{skip,{failed,Error}}},{Mod,Func},[]};
+ NewResult = do_end_tc_call(Mod,Func,{Error,Args0},
+ {skip,{failed,Error}}),
+ {{0,NewResult},{Mod,Func},[]};
{fail,Reason} ->
[Conf] = Args0,
Conf1 = [{tc_status,{failed,Reason}} | Conf],
fw_error_notify(Mod, Func, Conf, Reason),
- test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,
- {{error,Reason},[Conf1]}]),
- {{0,{failed,Reason}},{Mod,Func},[]};
+ NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf1]},
+ {fail, Reason}),
+ {{0,NewResult},{Mod,Func},[]};
Skip = {skip,_Reason} ->
- test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Skip,Args0}]),
- {{0,Skip},{Mod,Func},[]};
+ NewResult = do_end_tc_call(Mod,Func,{Skip,Args0},Skip),
+ {{0,NewResult},{Mod,Func},[]};
{auto_skip,Reason} ->
- test_server_sup:framework_call(end_tc,[?pl2a(Mod),
- Func,
- {{skip,Reason},Args0}]),
- {{0,{skip,{fw_auto_skip,Reason}}},{Mod,Func},[]}
+ NewResult = do_end_tc_call(Mod, Func, {{skip,Reason},Args0},
+ {skip, {fw_auto_skip,Reason}}),
+ {{0,NewResult},{Mod,Func},[]}
end,
exit({Ref,Time,Value,Loc,Opts}).
@@ -1103,14 +1098,14 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
Skip = {skip,Reason} ->
Line = get_loc(),
Conf = [{tc_status,{skipped,Reason}}],
- test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Skip,[Conf]}]),
- {{0,{skip,Reason}},Line,[]};
+ NewRes = do_end_tc_call(Mod,Func,{Skip,[Conf]}, Skip),
+ {{0,NewRes},Line,[]};
{skip_and_save,Reason,SaveCfg} ->
Line = get_loc(),
Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}],
- test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,
- {{skip,Reason},[Conf]}]),
- {{0,{skip,Reason}},Line,[]};
+ NewRes = do_end_tc_call(Mod, Func, {{skip, Reason}, [Conf]},
+ {skip, Reason}),
+ {{0,NewRes},Line,[]};
{ok,NewConf} ->
put(test_server_init_or_end_conf,undefined),
%% call user callback function if defined
@@ -1155,13 +1150,12 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
{FWReturn,TSReturn,EndConf1}
end,
put(test_server_init_or_end_conf,undefined),
- case test_server_sup:framework_call(end_tc, [?pl2a(Mod), Func,
- {FWReturn1,[EndConf2]}]) of
- {fail,Reason} ->
- fw_error_notify(Mod, Func, EndConf2, Reason),
- {{T,{failed,Reason}},{Mod,Func},[]};
- _ ->
- {{T,TSReturn1},Loc,[]}
+ case do_end_tc_call(Mod, Func, {FWReturn1,[EndConf2]}, TSReturn1) of
+ {failed,Reason} = NewReturn ->
+ fw_error_notify(Mod,Func,EndConf2, Reason),
+ {{T,NewReturn},{Mod,Func},[]};
+ NewReturn ->
+ {{T,NewReturn},Loc,[]}
end
end;
skip_init ->
@@ -1179,10 +1173,36 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
{{T,Return},Loc} = {ts_tc(Mod, Func, Args2),get_loc()},
%% call user callback function if defined
Return1 = user_callback(TCCallback, Mod, Func, 'end', Return),
- {Return2,Opts} = process_return_val([Return1], Mod,Func,Args1, Loc, Return1),
+ {Return2,Opts} = process_return_val([Return1], Mod, Func,
+ Args1, Loc, Return1),
{{T,Return2},Loc,Opts}
end.
+do_end_tc_call(M,F,Res,Return) ->
+ Ref = make_ref(),
+ case test_server_sup:framework_call(
+ end_tc, [?pl2a(M),F,Res], Ref) of
+ {fail,FWReason} ->
+ {failed,FWReason};
+ Ref ->
+ case test_server_sup:framework_call(
+ end_tc, [?pl2a(M),F,Res, Return], ok) of
+ {fail,FWReason} ->
+ {failed,FWReason};
+ ok ->
+ case Return of
+ {fail,Reason} ->
+ {failed,Reason};
+ Return ->
+ Return
+ end;
+ NewReturn ->
+ NewReturn
+ end;
+ _ ->
+ Return
+ end.
+
%% the return value is a list and we have to check if it contains
%% the result of an end conf case or if it's a Config list
process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) ->
@@ -1197,13 +1217,13 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) ->
end, Return) of
true -> % must be return value from end conf case
process_return_val1(Return, M,F,A, Loc, Final, []);
- false -> % must be Config value from init conf case
- case test_server_sup:framework_call(end_tc, [?pl2a(M),F,{ok,A}]) of
- {fail,FWReason} ->
+ false -> % must be Config value from init conf case
+ case do_end_tc_call(M,F,{ok,A}, Return) of
+ {failed, FWReason} = Failed ->
fw_error_notify(M,F,A, FWReason),
- {{failed,FWReason},[]};
- _ ->
- {Return,[]}
+ {Failed, []};
+ NewReturn ->
+ {NewReturn, []}
end
end;
%% the return value is not a list, so it's the return value from an
@@ -1211,16 +1231,16 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) ->
process_return_val(Return, M,F,A, Loc, Final) ->
process_return_val1(Return, M,F,A, Loc, Final, []).
-process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts) when E=='EXIT';
- E==failed ->
+process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts)
+ when E=='EXIT';
+ E==failed ->
fw_error_notify(M,F,A, TCError, mod_loc(Loc)),
- case test_server_sup:framework_call(end_tc,
- [?pl2a(M),F,{{error,TCError},
- [[{tc_status,{failed,TCError}}|Args]]}]) of
- {fail,FWReason} ->
+ case do_end_tc_call(M,F,{{error,TCError},
+ [[{tc_status,{failed,TCError}}|Args]]}, Failed) of
+ {failed,FWReason} ->
{{failed,FWReason},SaveOpts};
- _ ->
- {Failed,SaveOpts}
+ NewReturn ->
+ {NewReturn,SaveOpts}
end;
process_return_val1([SaveCfg={save_config,_}|Opts], M,F,[Args], Loc, Final, SaveOpts) ->
process_return_val1(Opts, M,F,[[SaveCfg|Args]], Loc, Final, SaveOpts);
@@ -1234,11 +1254,11 @@ process_return_val1([RetVal={Tag,_}|Opts], M,F,A, Loc, _, SaveOpts) when Tag==sk
process_return_val1([_|Opts], M,F,A, Loc, Final, SaveOpts) ->
process_return_val1(Opts, M,F,A, Loc, Final, SaveOpts);
process_return_val1([], M,F,A, _Loc, Final, SaveOpts) ->
- case test_server_sup:framework_call(end_tc, [?pl2a(M),F,{Final,A}]) of
- {fail,FWReason} ->
+ case do_end_tc_call(M,F,{Final,A}, Final) of
+ {failed,FWReason} ->
{{failed,FWReason},SaveOpts};
- _ ->
- {Final,lists:reverse(SaveOpts)}
+ NewReturn ->
+ {NewReturn,lists:reverse(SaveOpts)}
end.
user_callback(undefined, _, _, _, Args) ->
@@ -1263,7 +1283,7 @@ init_per_testcase(Mod, Func, Args) ->
false -> code:load_file(Mod);
_ -> ok
end,
- %% init_per_testcase defined, returns new configuration
+%% init_per_testcase defined, returns new configuration
case erlang:function_exported(Mod,init_per_testcase,2) of
true ->
case catch my_apply(Mod, init_per_testcase, [Func|Args]) of
@@ -1306,8 +1326,8 @@ init_per_testcase(Mod, Func, Args) ->
{skip,{failed,{Mod,init_per_testcase,Other}}}
end;
false ->
- %% Optional init_per_testcase not defined
- %% keep quiet.
+%% Optional init_per_testcase not defined
+%% keep quiet.
[Config] = Args,
{ok, Config}
end.
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 1dc5646184..977a2b1785 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -3552,7 +3552,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Where,
{_,{'EXIT',Reason}} ->
progress(failed, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
- {_, {failed, Reason}} ->
+ {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed ->
progress(failed, Num, Mod, Func, Loc, Reason,
Time, Comment, Style);
{_, {Skip, Reason}} when Skip==skip; Skip==skipped ->