path: root/lib/common_test/src/ct_hooks.erl

%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 2004-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License");
%% you may not use this file except in compliance with the License.
%% You may obtain a copy of the License at
%%
%%     http://www.apache.org/licenses/LICENSE-2.0
%%
%% Unless required by applicable law or agreed to in writing, software
%% distributed under the License is distributed on an "AS IS" BASIS,
%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
%% See the License for the specific language governing permissions and
%% limitations under the License.
%%
%% %CopyrightEnd%
%%

%%% @doc Common Test Framework test execution control module.
%%%
%%% <p>This module is a proxy for calling and handling common test hooks.</p>

-module(ct_hooks).

%% API Exports
-export([init/1]).
-export([init_tc/3]).
-export([end_tc/5]).
-export([terminate/1]).
-export([on_tc_skip/2]).
-export([on_tc_fail/2]).

%% If you change this, remember to update ct_util:look -> stop clause as well.
-define(config_name, ct_hooks).

%% All of the hooks which are to be started by default. Remove by issuing
%% -enable_builtin_hooks false to when starting common test.
-define(BUILTIN_HOOKS,[#ct_hook_config{ module = cth_log_redirect,
					opts = [],
					prio = ctfirst }]).

-record(ct_hook_config, {id, module, prio, scope, opts = [], state = []}).

%% -------------------------------------------------------------------------
%% API Functions
%% -------------------------------------------------------------------------

%% @doc Called before any suites are started
-spec init(State :: term()) -> ok |
			       {fail, Reason :: term()}.
init(Opts) ->
    call(get_builtin_hooks(Opts) ++ get_new_hooks(Opts, undefined),
	 ok, init, []).

%% @doc Called after all suites are done.
-spec terminate(Hooks :: term()) ->
    ok.
terminate(Hooks) ->
    call([{HookId, fun call_terminate/3}
	  || #ct_hook_config{id = HookId} <- Hooks],
	 ct_hooks_terminate_dummy, terminate, Hooks),
    ok.

%% @doc Called as each test case is started. This includes all configuration
%% tests.
-spec init_tc(Mod :: atom(),
	      FuncSpec :: atom() | 
			  {ConfigFunc :: init_per_testcase | end_per_testcase,
			   TestCase :: atom()} |
			  {ConfigFunc :: init_per_group | end_per_group,
			   GroupName :: atom(),
			   Properties :: list()},
	      Args :: list()) ->
    NewConfig :: proplists:proplist() |
		 {skip, Reason :: term()} |
		 {auto_skip, Reason :: term()} |
		 {fail, Reason :: term()}.

init_tc(Mod, init_per_suite, Config) ->
    Info = try proplists:get_value(ct_hooks, Mod:suite(),[]) of
	       List when is_list(List) -> 
		   [{?config_name,List}];
	       CTHook when is_atom(CTHook) ->
		   [{?config_name,[CTHook]}]
	   catch error:undef ->
		   [{?config_name,[]}]
	   end,
    call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]);
init_tc(Mod, end_per_suite, Config) ->
    call(fun call_generic/3, Config, [pre_end_per_suite, Mod]);
init_tc(Mod, {init_per_group, GroupName, Properties}, Config) ->
    maybe_start_locker(Mod, GroupName, Properties),
    call(fun call_generic_fallback/3, Config,
         [pre_init_per_group, Mod, GroupName]);
init_tc(Mod, {end_per_group, GroupName, _}, Config) ->
    call(fun call_generic_fallback/3, Config,
         [pre_end_per_group, Mod, GroupName]);
init_tc(Mod, {init_per_testcase,TC}, Config) ->
    call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]);
init_tc(Mod, {end_per_testcase,TC}, Config) ->
    call(fun call_generic_fallback/3, Config, [pre_end_per_testcase, Mod, TC]);
init_tc(Mod, TC = error_in_suite, Config) ->
    call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]).

%% @doc Called as each test case is completed. This includes all configuration
%% tests.
-spec end_tc(Mod :: atom(),
	     FuncSpec :: atom() |  
			 {ConfigFunc :: init_per_testcase | end_per_testcase,
			  TestCase :: atom()} |
			 {ConfigFunc :: init_per_group | end_per_group,
			  GroupName :: atom(),
			  Properties :: list()},
	     Args :: list(),
	     Result :: term(),
	     Return :: term()) ->
    NewConfig :: proplists:proplist() |
		 {skip, Reason :: term()} |
		 {auto_skip, Reason :: term()} |
		 {fail, Reason :: term()} |
		 ok | '$ct_no_change'.

end_tc(Mod, init_per_suite, Config, _Result, Return) ->
    call(fun call_generic/3, Return, [post_init_per_suite, Mod, Config],
	 '$ct_no_change');
end_tc(Mod, end_per_suite, Config, Result, _Return) ->
    call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config],
	'$ct_no_change');
end_tc(Mod, {init_per_group, GroupName, _}, Config, _Result, Return) ->
    call(fun call_generic_fallback/3, Return,
         [post_init_per_group, Mod, GroupName, Config], '$ct_no_change');
end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) ->
    Res = call(fun call_generic_fallback/3, Result,
	       [post_end_per_group, Mod, GroupName, Config], '$ct_no_change'),
    maybe_stop_locker(Mod, GroupName, Properties),
    Res;
end_tc(Mod, {init_per_testcase,TC}, Config, Result, _Return) ->
    call(fun call_generic_fallback/3, Result,
         [post_init_per_testcase, Mod, TC, Config], '$ct_no_change');
end_tc(Mod, {end_per_testcase,TC}, Config, Result, _Return) ->
    call(fun call_generic_fallback/3, Result,
         [post_end_per_testcase, Mod, TC, Config], '$ct_no_change');
end_tc(Mod, TC = error_in_suite, Config, Result, _Return) ->
    call(fun call_generic_fallback/3, Result,
         [post_end_per_testcase, Mod, TC, Config], '$ct_no_change').


%% Case = TestCase | {TestCase,GroupName}
on_tc_skip(How, {Suite, Case, Reason}) ->
    call(fun call_cleanup/3, {How, Reason}, [on_tc_skip, Suite, Case]).

%% Case = TestCase | {TestCase,GroupName}
on_tc_fail(_How, {Suite, Case, Reason}) ->
    call(fun call_cleanup/3, Reason, [on_tc_fail, Suite, Case]).

%% -------------------------------------------------------------------------
%% Internal Functions
%% -------------------------------------------------------------------------
call_id(#ct_hook_config{ module = Mod, opts = Opts} = Hook, Config, Scope) ->
    Id = catch_apply(Mod,id,[Opts], make_ref()),
    {Config, Hook#ct_hook_config{ id = Id, scope = scope(Scope)}}.
	
call_init(#ct_hook_config{ module = Mod, opts = Opts, id = Id, prio = P} = Hook,
	  Config,_Meta) ->
    case Mod:init(Id, Opts) of
	{ok, NewState} when P =:= undefined ->
	    {Config, Hook#ct_hook_config{ state = NewState, prio = 0 } };
	{ok, NewState} ->
	    {Config, Hook#ct_hook_config{ state = NewState } };
	{ok, NewState, Prio} when P =:= undefined ->
	    %% Only set prio if not already set when installing hook
	    {Config, Hook#ct_hook_config{ state = NewState, prio = Prio } };
	{ok, NewState, _} ->
	    {Config, Hook#ct_hook_config{ state = NewState } };
	NewState -> %% Keep for backward compatability reasons
	    {Config, Hook#ct_hook_config{ state = NewState } }
    end.    

call_terminate(#ct_hook_config{ module = Mod, state = State} = Hook, _, _) ->
    catch_apply(Mod,terminate,[State], ok),
    {[],Hook}.

call_cleanup(#ct_hook_config{ module = Mod, state = State} = Hook,
	     Reason, [Function | Args]) ->
    NewState = catch_apply(Mod,Function, Args ++ [Reason, State],
			   State, true),
    {Reason, Hook#ct_hook_config{ state = NewState } }.

call_generic(Hook, Value, Meta) ->
    do_call_generic(Hook, Value, Meta, false).

call_generic_fallback(Hook, Value, Meta) ->
    do_call_generic(Hook, Value, Meta, true).

do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook,
                Value, [Function | Args], Fallback) ->
    {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State],
				       {Value,State}, Fallback),
    {NewValue, Hook#ct_hook_config{ state = NewState } }.

%% Generic call function
call(Fun, Config, Meta) ->
    maybe_lock(),
    Hooks = get_hooks(),
    Calls = get_new_hooks(Config, Fun) ++
	[{HookId,Fun} || #ct_hook_config{id = HookId} <- Hooks],
    Res = call(resort(Calls,Hooks,Meta),
	       remove(?config_name,Config), Meta, Hooks),
    maybe_unlock(),
    Res.

call(Fun, Config, Meta, NoChangeRet) when is_function(Fun) ->
    case call(Fun,Config,Meta) of
	Config -> NoChangeRet;
	NewReturn -> NewReturn
    end;

call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) ->
    try
	{Config, #ct_hook_config{ id = NewId } = NewHook} =
	    call_id(Hook, Config, Meta),
	{NewHooks, NewRest} = 
	    case lists:keyfind(NewId, #ct_hook_config.id, Hooks) of
		false when NextFun =:= undefined ->
		    {Hooks ++ [NewHook],
		     Rest ++ [{NewId, call_init}]};
		ExistingHook when is_tuple(ExistingHook) ->
		    {Hooks, Rest};
		_ ->
		    {Hooks ++ [NewHook],
		     Rest ++ [{NewId, call_init}, {NewId,NextFun}]}
	    end,
	call(resort(NewRest,NewHooks,Meta), Config, Meta, NewHooks)
    catch Error:Reason ->
	    Trace = erlang:get_stacktrace(),
	    ct_logs:log("Suite Hook","Failed to start a CTH: ~p:~p",
			[Error,{Reason,Trace}]),
	    call([], {fail,"Failed to start CTH"
		      ", see the CT Log for details"}, Meta, Hooks)
    end;
call([{HookId, call_init} | Rest], Config, Meta, Hooks) ->
    call([{HookId, fun call_init/3} | Rest], Config, Meta, Hooks);
call([{HookId, Fun} | Rest], Config, Meta, Hooks) ->
    try
        Hook = lists:keyfind(HookId, #ct_hook_config.id, Hooks),
        {NewConf, NewHook} =  Fun(Hook, Config, Meta),
        NewCalls = get_new_hooks(NewConf, Fun),
        NewHooks = lists:keyreplace(HookId, #ct_hook_config.id, Hooks, NewHook),
        call(resort(NewCalls ++ Rest,NewHooks,Meta), %% Resort if call_init changed prio
	     remove(?config_name, NewConf), Meta,
             terminate_if_scope_ends(HookId, Meta, NewHooks))
    catch throw:{error_in_cth_call,Reason} ->
            call(Rest, {fail, Reason}, Meta,
                 terminate_if_scope_ends(HookId, Meta, Hooks))
    end;
call([], Config, _Meta, Hooks) ->
    save_suite_data_async(Hooks),
    Config.

remove(Key,List) when is_list(List) ->
    [Conf || Conf <- List, is_tuple(Conf) =:= false
		 orelse element(1, Conf) =/= Key];
remove(_, Else) ->
    Else.

%% Translate scopes, i.e. is_tuplenit_per_group,group1 -> end_per_group,group1 etc
scope([pre_init_per_testcase, SuiteName, TC|_]) ->
    [post_init_per_testcase, SuiteName, TC];
scope([pre_end_per_testcase, SuiteName, TC|_]) ->
    [post_end_per_testcase, SuiteName, TC];
scope([pre_init_per_group, SuiteName, GroupName|_]) ->
    [post_end_per_group, SuiteName, GroupName];
scope([post_init_per_group, SuiteName, GroupName|_]) ->
    [post_end_per_group, SuiteName, GroupName];
scope([pre_init_per_suite, SuiteName|_]) ->
    [post_end_per_suite, SuiteName];
scope([post_init_per_suite, SuiteName|_]) ->
    [post_end_per_suite, SuiteName];
scope(init) ->
    none.

strip_config([post_init_per_testcase, SuiteName, TC|_]) ->
    [post_init_per_testcase, SuiteName, TC];
strip_config([post_end_per_testcase, SuiteName, TC|_]) ->
    [post_end_per_testcase, SuiteName, TC];
strip_config([post_init_per_group, SuiteName, GroupName|_]) ->
    [post_init_per_group, SuiteName, GroupName];
strip_config([post_end_per_group, SuiteName, GroupName|_]) ->
    [post_end_per_group, SuiteName, GroupName];
strip_config([post_init_per_suite, SuiteName|_]) ->
    [post_init_per_suite, SuiteName];
strip_config([post_end_per_suite, SuiteName|_]) ->
    [post_end_per_suite, SuiteName];
strip_config(Other) ->
    Other.


terminate_if_scope_ends(HookId, [on_tc_skip,Suite,{end_per_group,Name}],
			Hooks) ->
    terminate_if_scope_ends(HookId, [post_end_per_group, Suite, Name], Hooks);
terminate_if_scope_ends(HookId, [on_tc_skip,Suite,end_per_suite], Hooks) ->
    terminate_if_scope_ends(HookId, [post_end_per_suite, Suite], Hooks);
terminate_if_scope_ends(HookId, Function0, Hooks) ->
    Function = strip_config(Function0),
    case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of
        #ct_hook_config{ id = HookId, scope = Function} = Hook ->
            terminate([Hook]),
            lists:keydelete(HookId, #ct_hook_config.id, Hooks);
        _ ->
            Hooks
    end.

%% Fetch hook functions
get_new_hooks(Config, Fun) ->
    lists:map(fun(NewHook) when is_atom(NewHook) ->
		      {#ct_hook_config{ module = NewHook }, call_id, Fun};
		 ({NewHook,Opts}) ->
		      {#ct_hook_config{ module = NewHook,
					opts = Opts}, call_id, Fun};
		 ({NewHook,Opts,Prio}) ->
		      {#ct_hook_config{ module = NewHook,
					opts = Opts,
					prio = Prio }, call_id, Fun}
		end, get_new_hooks(Config)).

get_new_hooks(Config) when is_list(Config) ->
    lists:flatmap(fun({?config_name, HookConfigs}) when is_list(HookConfigs) ->
			  HookConfigs;
		     ({?config_name, HookConfig}) when is_atom(HookConfig) ->
			  [HookConfig];
		     (_) ->
			  []
		  end, Config);
get_new_hooks(_Config) ->
    [].

get_builtin_hooks(Opts) ->
    case proplists:get_value(enable_builtin_hooks,Opts) of
	false ->
	    [];
	_Else ->
	    [{HookConf, call_id, undefined} || HookConf <- ?BUILTIN_HOOKS]
    end.

save_suite_data_async(Hooks) ->
    ct_util:save_suite_data_async(?config_name, Hooks).

get_hooks() ->
    lists:keysort(#ct_hook_config.prio,ct_util:read_suite_data(?config_name)).

%% Sort all calls in this order:
%% call_id < call_init < ctfirst < Priority 1 < .. < Priority N < ctlast
%% If Hook Priority is equal, check when it has been installed and
%% sort on that instead.
%% If we are doing a cleanup call i.e. {post,pre}_end_per_*, all priorities
%% are reversed. Probably want to make this sorting algorithm pluginable
%% as some point...
resort(Calls,Hooks,[F|_R]) when F == pre_end_per_testcase;
				F == post_end_per_testcase;
				F == pre_end_per_group;
				F == post_end_per_group;
				F == pre_end_per_suite;
				F == post_end_per_suite ->
    lists:reverse(resort(Calls,Hooks));
resort(Calls,Hooks,_Meta) ->
    resort(Calls,Hooks).
    
resort(Calls, Hooks) ->
    lists:sort(
      fun({_,_,_},_) ->
	      true;
	 (_,{_,_,_}) ->
	      false;
	 ({_,call_init},_) ->
	      true;
	 (_,{_,call_init}) ->
	      false;
	 ({Id1,_},{Id2,_}) ->
	      P1 = (lists:keyfind(Id1, #ct_hook_config.id, Hooks))#ct_hook_config.prio,
	      P2 = (lists:keyfind(Id2, #ct_hook_config.id, Hooks))#ct_hook_config.prio,
	      if
		  P1 == P2 ->
		      %% If priorities are equal, we check the position in the
		      %% hooks list
		      pos(Id1,Hooks) < pos(Id2,Hooks);
		  P1 == ctfirst ->
		      true;
		  P2 == ctfirst ->
		      false;
		  P1 == ctlast ->
		      false;
		  P2 == ctlast ->
		      true;
		  true ->
		      P1 < P2
	      end
      end,Calls).

pos(Id,Hooks) ->
    pos(Id,Hooks,0).
pos(Id,[#ct_hook_config{ id = Id}|_],Num) ->
    Num;
pos(Id,[_|Rest],Num) ->
    pos(Id,Rest,Num+1).


catch_apply(M,F,A, Default) ->
    catch_apply(M,F,A,Default,false).
catch_apply(M,F,A, Default, Fallback) ->
    not erlang:module_loaded(M) andalso (catch M:module_info()),
    case erlang:function_exported(M,F,length(A)) of
        false when Fallback ->
            catch_apply(M,F,tl(A),Default,false);
        false ->
            Default;
        true ->
            catch_apply(M,F,A)
    end.

catch_apply(M,F,A) ->
    try
        erlang:apply(M,F,A)
    catch _:Reason ->
            Trace = erlang:get_stacktrace(),
            ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p",
                            [error,{Reason,Trace}]),
            throw({error_in_cth_call,
                   lists:flatten(
                     io_lib:format("~w:~w/~w CTH call failed",
                                   [M,F,length(A)]))})
    end.


%% We need to lock around the state for parallel groups only. This is because
%% we will get several processes reading and writing the state for a single
%% cth at the same time.
maybe_start_locker(Mod,GroupName,Opts) ->
    case lists:member(parallel,Opts) of
	true ->
	    {ok, _Pid} = ct_hooks_lock:start({Mod,GroupName}),
	    ok;
	false ->
	    ok
    end.

maybe_stop_locker(Mod,GroupName,Opts) ->
    case lists:member(parallel,Opts) of
	true ->
	    stopped = ct_hooks_lock:stop({Mod,GroupName});
	false ->
	    ok
    end.


maybe_lock() ->
    locked = ct_hooks_lock:request().

maybe_unlock() ->
    unlocked = ct_hooks_lock:release().