%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2003-2009. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%%
%% %CopyrightEnd%
%%
%%----------------------------------------------------------------------
%% Purpose: Lightweight test server
%%----------------------------------------------------------------------
-module(snmp_test_server).
-compile(export_all).
-export([
run/1, run/2,
error/3,
skip/3,
fatal_skip/3,
init_per_testcase/2,
fin_per_testcase/2
]).
-include("snmp_test_lib.hrl").
-define(GLOBAL_LOGGER, snmp_global_logger).
-define(TEST_CASE_SUP, snmp_test_case_supervisor).
-define(d(F,A),d(F,A,?LINE)).
-ifndef(snmp_priv_dir).
-define(snmp_priv_dir, "run-" ++ timestamp()).
-endif.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Evaluates a test case or test suite
%% Returns a list of failing test cases:
%%
%% {Mod, Fun, ExpectedRes, ActualRes}
%%----------------------------------------------------------------------
run([Mod, Fun]) when is_atom(Mod) andalso is_atom(Fun) ->
Res = run({Mod, Fun}, default_config(Mod)),
display_result(Res),
Res;
run({Mod, _Fun} = Case) when is_atom(Mod) ->
io:format("~n", []),
Res = run(Case, default_config(Mod)),
display_result(Res),
Res;
run(Mod) when is_atom(Mod) ->
io:format("~n", []),
Res = run(Mod, default_config(Mod)),
display_result(Res),
Res;
run([Mod]) when is_atom(Mod) ->
io:format("~n", []),
Res = run(Mod, default_config(Mod)),
display_result(Res),
Res.
run({Mod, Fun}, Config) when is_atom(Mod) andalso
is_atom(Fun) andalso
is_list(Config) ->
?d("run(~p,~p) -> entry", [Mod, Fun]),
case (catch apply(Mod, Fun, [suite])) of
[] ->
io:format("~n~n*** Eval: ~p ***************~n",
[{Mod, Fun}]),
case eval(Mod, Fun, Config) of
{ok, _, _} ->
[];
Other ->
[Other]
end;
Cases when is_list(Cases) ->
io:format("~n*** Expand: ~p ...~n", [{Mod, Fun}]),
Map = fun(Case) when is_atom(Case) -> {Mod, Case};
(Case) -> Case
end,
run(lists:map(Map, Cases), Config);
{conf, InitSuite, Cases, FinishSuite} when is_atom(InitSuite) andalso
is_list(Cases) andalso
is_atom(FinishSuite) ->
?d("run -> conf: "
"~n InitSuite: ~p"
"~n Cases: ~p"
"~n FinishSuite: ~p", [InitSuite, Cases, FinishSuite]),
do_suite(Mod, InitSuite, Cases, FinishSuite, Config);
{req, _, SubCases} when is_list(SubCases) ->
?d("run -> req: "
"~n SubCases: ~p", [SubCases]),
do_subcases(Mod, Fun, SubCases, Config, []);
{req, _, Conf} ->
?d("run -> req: "
"~n Conf: ~p", [Conf]),
do_subcases(Mod, Fun, [Conf], Config, []);
{'EXIT', {undef, _}} ->
io:format("~n*** Undefined: ~p~n", [{Mod, Fun}]),
[{nyi, {Mod, Fun}, ok}];
Error ->
io:format("~n*** Ignoring: ~p: ~p~n", [{Mod, Fun}, Error]),
[{failed, {Mod, Fun}, Error}]
end;
run(Mod, Config) when is_atom(Mod) andalso is_list(Config) ->
run({Mod, all}, Config);
run(Cases, Config) when is_list(Cases) andalso is_list(Config) ->
Errors = [run(Case, Config) || Case <- Cases],
lists:append(Errors);
run(Bad, _Config) ->
[{badarg, Bad, ok}].
do_suite(Mod, Init, Cases, Finish, Config0) ->
?d("do_suite -> entry with"
"~n Mod: ~p"
"~n Init: ~p"
"~n Cases: ~p"
"~n Finish: ~p"
"~n Config0: ~p", [Mod, Init, Cases, Finish, Config0]),
case (catch apply(Mod, Init, [Config0])) of
Config when is_list(Config) ->
io:format("~n*** Expand: ~p ...~n", [Mod]),
Map = fun(Case) when is_atom(Case) -> {Mod, Case};
(Case) -> Case
end,
Res = run(lists:map(Map, Cases), Config),
(catch apply(Mod, Finish, [Config])),
Res;
{'EXIT', {skipped, Reason}} ->
io:format(" => skipping: ~p~n", [Reason]),
SkippedCases =
[{skipped, {Mod, Case}, suite_init_skipped} || Case <- Cases],
lists:flatten([[{skipped, {Mod, Init}, Reason}],
SkippedCases,
[{skipped, {Mod, Finish}, suite_init_skipped}]]);
Error ->
io:format(" => init (~p) failed: ~n~p~n", [Init, Error]),
InitResult =
[{failed, {Mod, Init}, Error}],
SkippedCases =
[{skipped, {Mod, Case}, suite_init_failed} || Case <- Cases],
FinResult =
case (catch apply(Mod, Finish, [Config0])) of
ok ->
[];
FinConfig when is_list(FinConfig) ->
[];
FinError ->
[{failed, {Mod, Finish}, FinError}]
end,
lists:flatten([InitResult, SkippedCases, FinResult])
end.
do_subcases(_Mod, _Fun, [], _Config, Acc) ->
lists:flatten(lists:reverse(Acc));
do_subcases(Mod, Fun, [{conf, Init, Cases, Finish}|SubCases], Config, Acc)
when is_atom(Init) andalso is_list(Cases) andalso is_atom(Finish) ->
R = case (catch apply(Mod, Init, [Config])) of
Conf when is_list(Conf) ->
io:format("~n*** Expand: ~p ...~n", [{Mod, Fun}]),
Map = fun(Case) when is_atom(Case) -> {Mod, Case};
(Case) -> Case
end,
Res = run(lists:map(Map, Cases), Conf),
(catch apply(Mod, Finish, [Conf])),
Res;
{'EXIT', {skipped, Reason}} ->
io:format(" => skipping: ~p~n", [Reason]),
[{skipped, {Mod, Fun}, Reason}];
Error ->
io:format(" => init (~p) failed: ~n~p~n", [Init, Error]),
(catch apply(Mod, Finish, [Config])),
[{failed, {Mod, Fun}, Error}]
end,
do_subcases(Mod, Fun, SubCases, Config, [R|Acc]);
do_subcases(Mod, Fun, [SubCase|SubCases], Config, Acc) when atom(SubCase) ->
R = do_case(Mod, SubCase, Config),
do_subcases(Mod, Fun, SubCases,Config, [R|Acc]).
do_case(M, F, C) ->
io:format("~n~n*** Eval: ~p ***************~n", [{M, F}]),
case eval(M, F, C) of
{ok, _, _} ->
[];
Other ->
[Other]
end.
eval(Mod, Fun, Config) ->
Flag = process_flag(trap_exit, true),
global:register_name(?TEST_CASE_SUP, self()),
Config2 = Mod:init_per_testcase(Fun, Config),
Self = self(),
Eval = fun() -> do_eval(Self, Mod, Fun, Config2) end,
Pid = spawn_link(Eval),
R = wait_for_evaluator(Pid, Mod, Fun, Config2, []),
Mod:fin_per_testcase(Fun, Config2),
global:unregister_name(?TEST_CASE_SUP),
process_flag(trap_exit, Flag),
R.
wait_for_evaluator(Pid, Mod, Fun, Config, Errors) ->
Pre = lists:concat(["TEST CASE: ", Fun]),
receive
{'EXIT', _Watchdog, watchdog_timeout} ->
io:format("*** ~s WATCHDOG TIMEOUT~n", [Pre]),
exit(Pid, kill),
{failed, {Mod, Fun}, watchdog_timeout};
{done, Pid, ok} when Errors =:= [] ->
io:format("*** ~s OK~n", [Pre]),
{ok, {Mod, Fun}, Errors};
{done, Pid, {ok, _}} when Errors =:= [] ->
io:format("*** ~s OK~n", [Pre]),
{ok, {Mod, Fun}, Errors};
{done, Pid, Fail} ->
io:format("*** ~s FAILED~n~p~n", [Pre, Fail]),
{failed, {Mod, Fun}, Fail};
{'EXIT', Pid, {skipped, Reason}} ->
io:format("*** ~s SKIPPED~n~p~n", [Pre, Reason]),
{skipped, {Mod, Fun}, Errors};
{'EXIT', Pid, Reason} ->
io:format("*** ~s CRASHED~n~p~n", [Pre, Reason]),
{crashed, {Mod, Fun}, [{'EXIT', Reason} | Errors]};
{fail, Pid, Reason} ->
io:format("*** ~s FAILED~n~p~n", [Pre, Reason]),
wait_for_evaluator(Pid, Mod, Fun, Config, Errors ++ [Reason])
end.
do_eval(ReplyTo, Mod, Fun, Config) ->
case (catch apply(Mod, Fun, [Config])) of
{'EXIT', {skipped, Reason}} ->
ReplyTo ! {'EXIT', self(), {skipped, Reason}};
Other ->
ReplyTo ! {done, self(), Other}
end,
unlink(ReplyTo),
exit(shutdown).
display_result([]) ->
io:format("TEST OK~n", []);
display_result(Errors) when is_list(Errors) ->
Nyi = [MF || {nyi, MF, _} <- Errors],
Skipped = [{MF, Reason} || {skipped, MF, Reason} <- Errors],
Crashed = [{MF, Reason} || {crashed, MF, Reason} <- Errors],
Failed = [{MF, Reason} || {failed, MF, Reason} <- Errors],
display_skipped(Skipped),
display_crashed(Crashed),
display_failed(Failed),
display_summery(Nyi, Skipped, Crashed, Failed).
display_summery(Nyi, Skipped, Crashed, Failed) ->
io:format("~nTest case summery:~n", []),
display_summery(Nyi, "not yet implemented"),
display_summery(Skipped, "skipped"),
display_summery(Crashed, "crashed"),
display_summery(Failed, "failed"),
io:format("~n", []).
display_summery([], _) ->
ok;
display_summery(Res, Info) ->
io:format(" ~w test cases ~s~n", [length(Res), Info]).
display_skipped([]) ->
io:format("Skipped test cases: -~n", []);
display_skipped(Skipped) ->
io:format("Skipped test cases:~n", []),
[io:format(" ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Skipped],
io:format("~n", []).
display_crashed([]) ->
io:format("Crashed test cases: -~n", []);
display_crashed(Crashed) ->
io:format("Crashed test cases:~n", []),
[io:format(" ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Crashed],
io:format("~n", []).
display_failed([]) ->
io:format("Failed test cases: -~n", []);
display_failed(Failed) ->
io:format("Failed test cases:~n", []),
[io:format(" ~p => ~p~n", [MF, Reason]) || {MF, Reason} <- Failed],
io:format("~n", []).
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Verify that the actual result of a test case matches the exected one
%% Returns the actual result
%% Stores the result in the process dictionary if mismatch
error(Actual, Mod, Line) ->
global:send(?GLOBAL_LOGGER, {failed, Mod, Line}),
log("<ERROR> Bad result: ~p~n", [Actual], Mod, Line),
case global:whereis_name(?TEST_CASE_SUP) of
undefined ->
ignore;
Pid ->
Pid ! {fail, self(), {Actual, Mod, Line}}
end,
Actual.
skip(Actual, Mod, Line) ->
log("Skipping test case~n", [], Mod, Line),
exit({skipped, {Actual, Mod, Line}}).
fatal_skip(Actual, Mod, Line) ->
error(Actual, Mod, Line),
exit(shutdown).
log(Format, Args, Mod, Line) ->
case global:whereis_name(?GLOBAL_LOGGER) of
undefined ->
io:format(user, "~p(~p): " ++ Format, [Mod, Line] ++ Args);
Pid ->
io:format(Pid, "~p(~p): " ++ Format, [Mod, Line] ++ Args)
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Test server callbacks
init_per_testcase(_Case, Config) ->
global:register_name(?GLOBAL_LOGGER, group_leader()),
Config.
fin_per_testcase(_Case, _Config) ->
global:unregister_name(?GLOBAL_LOGGER),
ok.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% Internal utility functions
default_config(Mod) ->
PrivDir0 = ?snmp_priv_dir,
case filename:pathtype(PrivDir0) of
absolute ->
ok;
_ ->
case file:make_dir(Mod) of
ok ->
ok;
{error, eexist} ->
ok
end,
PrivDir = filename:join(Mod, PrivDir0),
case file:make_dir(PrivDir) of
ok ->
ok;
{error, eexist} ->
ok;
Error ->
?FAIL({failed_creating_subsuite_top_dir, Error})
end,
[{priv_dir, PrivDir}]
end.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
d(F, A, L) ->
d(true, F, A, L).
%% d(get(dbg), F, A, L).
d(true, F, A, L) ->
io:format("STS[~w] ~p " ++ F ++ "~n", [L,self()|A]);
d(_, _, _, _) ->
ok.
timestamp() ->
{Date, Time} = calendar:now_to_datetime( now() ),
{YYYY, MM, DD} = Date,
{Hour, Min, Sec} = Time,
FormatDate =
io_lib:format("~.4w-~.2.0w-~.2.0w_~.2.0w.~.2.0w.~.2.0w",
[YYYY,MM,DD,Hour,Min,Sec]),
lists:flatten(FormatDate).
