path: root/lib/inviso/test/inviso_tool_SUITE.erl

%%
%% %CopyrightBegin%
%%
%% Copyright Ericsson AB 1997-2010. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
%% compliance with the License. You should have received a copy of the
%% Erlang Public License along with this software. If not, it can be
%% retrieved online at http://www.erlang.org/.
%%
%% Software distributed under the License is distributed on an "AS IS"
%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
%% the License for the specific language governing rights and limitations
%% under the License.
%%
%% %CopyrightEnd%
%%
%%
%% Description:
%% Test suite for the inviso_tool. It is here assumed that inviso works
%% properly.
%%
%% Authors:
%% Lennart �hman, [email protected]
%% -----------------------------------------------------------------------------

-module(inviso_tool_SUITE).
-compile(export_all).

-include_lib("common_test/include/ct.hrl").
-include_lib("kernel/include/file.hrl").

-define(l,?line).

suite() -> [{ct_hooks,[ts_install_cth]}].

all() -> 
    [dist_basic_1, dist_rtc, dist_reconnect, dist_adopt,
     dist_history, dist_start_session_special].

groups() -> 
    [].

init_per_group(_GroupName, Config) ->
    Config.

end_per_group(_GroupName, Config) ->
    Config.


%% -----------------------------------------------------------------------------

init_per_suite(Config) ->
    Config.
%% -----------------------------------------------------------------------------

end_per_suite(_Config) ->
    ok.
%% -----------------------------------------------------------------------------

%% For each distributed testcase, we need two other distributed nodes to run the
%% runtime components on. Since they are freshly started every time there is no
%% need to clean them up first.
init_per_testcase(_Case,Config) ->
    ?l TH=test_server:timetrap(100000),
    ?l {ok,Node1}=test_server:start_node(inviso1,peer,[]),
    ?l {ok,Node2}=test_server:start_node(inviso2,peer,[]),
    ?l SuiteDir=filename:dirname(code:which(?MODULE)),

    %% Otherwise peer nodes will not find this module!
    ?l true=rpc:call(Node1,code,add_patha,[SuiteDir]),
    ?l true=rpc:call(Node2,code,add_patha,[SuiteDir]),

    %% SPECIAL FOR MY PRIVATE TEST ENVIROMENT
%    ?l rpc:call(Node1,code,add_patha,["/clearcase/otp/tools/runtime_tools/ebin"]),
%    ?l rpc:call(Node1,code,add_patha,["/clearcase/otp/tools/inviso/ebin"]),
%    ?l rpc:call(Node2,code,add_patha,["/clearcase/otp/tools/runtime_tools/ebin"]),
%    ?l rpc:call(Node2,code,add_patha,["/clearcase/otp/tools/inviso/ebin"]),

%    %% SPECIAL FOR MY PRIVATE TEST ENVIROMENT, windows.
%    ?l rpc:call(Node1,code,add_patha,["C:/DATA/PROJECTS/inviso_project/runtime_tools/ebin"]),
%    ?l rpc:call(Node1,code,add_patha,["C:/DATA/PROJECTS/inviso_project/inviso/ebin"]),
%    ?l rpc:call(Node2,code,add_patha,["C:/DATA/PROJECTS/inviso_project/runtime_tools/ebin"]),
%    ?l rpc:call(Node2,code,add_patha,["C:/DATA/PROJECTS/inviso_project/inviso/ebin"]),

    ?l ok=rpc:call(Node1,application,start,[runtime_tools]),
    ?l ok=rpc:call(Node2,application,start,[runtime_tools]),
    ?l timer:sleep(100),                     % Problem with autostarted runtime.
    %% The following is a test that the inviso_rt processes which are autostarted
    %% are now gone.

    ?l ok=poll(rpc,call,[Node1,erlang,whereis,[inviso_rt]],undefined,20),
    ?l ok=poll(rpc,call,[Node2,erlang,whereis,[inviso_rt]],undefined,20),

    NewConfig1=insert_remotenode_config(inviso1,Node1,Config),
    NewConfig2=insert_remotenode_config(inviso2,Node2,NewConfig1),
    insert_timetraphandle_config(TH,NewConfig2).
%% -----------------------------------------------------------------------------

end_per_testcase(_Case,Config) ->
    ?l test_server:stop_node(get_remotenode_config(inviso1,Config)),
    ?l test_server:stop_node(get_remotenode_config(inviso2,Config)),
    ?l test_server:timetrap_cancel(get_timetraphandle_config(Config)),
    ?l case whereis(inviso_tool) of          % In case inviso_tool did not stop.
	   Pid when pid(Pid) ->
	       ?l io:format("Had to kill inviso_tool!~n",[]),
	       ?l exit(Pid,kill);
	   _ ->
	       true
       end,
    ?l case whereis(inviso_rt) of            % In case we ran a runtime here.
	   Pid2 when pid(Pid2) ->
	       ?l io:format("Had to kill inviso_rt!~n",[]),
	       ?l exit(Pid2,kill);
	   _ ->
	       true
       end,
    ?l case whereis(inviso_c) of             % In case we ran the controll component here.
	   Pid3 when pid(Pid3) ->
	       ?l io:format("Had to kill inviso_c!~n",[]),
	       ?l exit(Pid3,kill);
	   _ ->
	       true
       end,
    NewConfig1=remove_remotenode_config(inviso1,Config),
    NewConfig2=remove_remotenode_config(inviso2,NewConfig1),
    remove_timetraphandle_config(NewConfig2).
%% -----------------------------------------------------------------------------

%% ==============================================================================
%% Testcases.
%% ==============================================================================

%% -----------------------------------------------------------------------------
%% Functional tests:
%% API:
%%   start/0                                 dist_basic_1
%%   stop/0                                  dist_basic_1
%%   reconnect_nodes/1                       dist_reconnect
%%   start_session/0                         dist_basic_1
%%   reinitiate_session/1                    dist_reconnect
%%   stop_session/0                          dist_basic_1
%%   atc/3                                   dist_basic_1
%%   sync_atc/3                              dist_basic_1
%%   sync_rtc/2,                             dist_rtc
%%   dtc/2                                   dist_basic_1
%%   sync_dtc/2                              dist_basic_1
%%   inviso/2                                dist_basic_1
%%   reactivate/1                            dist_basic_1
%%   get_autostart_data/2                    dist_basic_1
%%   get_activities/0                        dist_basic_1
%%   save_history/1                          dist_history
%%   restore_session/1                       dist_history
%%   get_node_status/1                       dist_basic_1
%%   get_session_data/0                      dist_basic_1
%%   flush/0                                 dist_basic_1
%% -----------------------------------------------------------------------------

%% Non functional tests:
%%   Run the control component on both the   dist_history
%%     same node as the tool and on a
%%     different node.
%%   Let a trace case crash in its execution dist_basic_1
%%     and check that it does not become
%%     part of the history.
%%   Check that tracer data becomes what the NOT IMPLEMENTED
%%     tdg function generates.
%%   Check that all inviso runtime           stop_inviso_tool/2
%%     components terminate if the tool is
%%     killed.
%%   Check that activation/deactivation      dist_basic_1
%%     cancels each other out in the history.
%%   Check that regexp expansion is done on  dist_reconnect
%%     another node if regexp_node is down.
%%   Test that tool-commands activating      dist_basic_1
%%     something done during a reactivation
%%     are actually done a bit later at the
%%     reactivated node (this since the the
%%     command being reactivated at the momen
%%     at the reactivating node may not
%%     be finished at the time the new tool
%%     command is issued).
%%   Check that deactivating tracecases are  dist_basic_1
%%     not redone at a reactivating node.
%%     (to prevent it from being redone and
%%     then just deactivated).
%%   Check that on-going reactivators and    NOT IMPLEMENTED
%%     tracecases are killed when stop_session.
%%   Check that inviso_tool can and will adopt
%%     a running runtime component.          dist_adopt
%% -----------------------------------------------------------------------------

-define(TC_DEF_FILE,filename:join(DataDir,"tracecase_def.txt")).

%% TEST CASE: Basic, distributed, start of inviso_tool with simple tracing.
dist_basic_1(doc) -> ["Simple test"];
dist_basic_1(suite) -> [];
dist_basic_1(Config) when list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=node(),
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=filename:join(?config(priv_dir,Config),""),
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l start_inviso_tool(Nodes,CNode,Opts),
    %% Now we know that all inviso runtimes are running and are not tracing.
    ?l {error,no_session}=inviso_tool:inviso(tpl,[lists,module_info,0,[]]),
    ?l {error,no_session}=inviso_tool:get_session_data(),
    ?l start_inviso_tool_session(CNode,[],1,Nodes),
    ?l {ok,{tracing,1,TDGargs}}=inviso_tool:get_session_data(),
    ?l true=is_list(TDGargs),
    %% Check that the initial tracecase has been executed at all tracing nodes.
    ?l lists:foreach(fun(N)->
			     ok=poll(rpc,
				     call,
				     [N,
				      erlang,
				      trace_info,
				      [{lists,module_info,1},traced]],
				     {traced,local},
				     20)
		     end,
		     Nodes),
    %% Start a test process at every node with a runtime component.
    ?l lists:foreach(fun(N)->spawn(N,?MODULE,test_proc_init,[]) end,Nodes),

    %% Let the processes start
    timer:sleep(100),

    %% Find the pids of the test processes.
    ?l TestProcs=lists:map(fun(N)->rpc:call(N,erlang,whereis,[inviso_tool_test_proc]) end,
			   Nodes),
    ?l true=(1=<length(TestProcs)),

    %% Activate a trace case.
    ?l {error,unknown_tracecase}=inviso_tool:atc(nonexistant,id,[]),
    ?l {error,{missing_variable,'ProcessName'}}=
	inviso_tool:atc(tracecase1,id1,[]),
    ?l ok=inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),
    ?l {error,activating}=inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),
    ?l {ok,[{tracecases,[{{tracecase1,id1},activating}]}]}=inviso_tool:get_activities(),
    ?l timer:sleep(1700),                     % There is a 500 ms delay in the tracecase.
    ?l {error,already_started}=
	inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),

    %% Now check that the trace case was executed at Nodes.
    ?l lists:foreach(fun(P)->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[P,flags]],
					{flags,[call]},
					10),
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,1},traced]],
					{traced,local},
					10)
		     end,
		     TestProcs),
    ?l lists:foreach(fun(P) ->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,0},traced]],
					{traced,false},
					1)
		     end,
		     TestProcs),

    %% Test inviso_tool:inviso/2.
    ?l {ok,NodeResults1}=inviso_tool:inviso(tpl,[math,module_info,0,[]]),
    ?l true=check_noderesults(Nodes,{ok,[1]},NodeResults1),
    ?l lists:foreach(fun(P) ->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,0},traced]],
					{traced,local},
					10)
		     end,
		     TestProcs),

    %% Test inviso_tool:sync_atc/3.
    ?l a_return_value=inviso_tool:sync_atc(tracecase2,id2,[]), % This will take 3000 ms.
    ?l lists:foreach(fun(P) ->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,pi,0},traced]],
					{traced,local},
					10)
		     end,
		     TestProcs),

    %% Test the reactivation mechanism.
    ?l [ANode|OtherNodes]=Nodes,             % Get a node to suspend.
    ?l {ok,{tracing,running}}=inviso_tool:get_node_status(ANode),
    ?l {ok,[{ANode,ok}]}=rpc:call(CNode,inviso,suspend,[[ANode],test]),
    ?l [APid|_]=TestProcs,                   % The first process is at ANode.
    %% Now check that trace flags were removed at ANode. This is actually testing inviso.
    ?l ok=poll(rpc,call,[node(APid),erlang,trace_info,[APid,flags]],{flags,[]},10),
    ?l {ok,{tracing,suspended}}=inviso_tool:get_node_status(ANode),
    %% Now reactivate it and expect the history to be redone. But it will take
    %% 3000 ms since there is a delay in tracecase2. Use that delay to issue a new
    %% tool command.
    ?l ok=inviso_tool:reactivate(ANode),
    ?l {ok,reactivating}=inviso_tool:get_node_status(ANode),
    ?l {ok,NodeResults2}=inviso_tool:inviso(tpl,[math,sin,1,[]]),
    ?l true=check_noderesults(OtherNodes,{ok,[1]},NodeResults2),
    %% Verify that the inviso command was not done (yet) at ANode.
    ?l {traced,false}=rpc:call(ANode,erlang,trace_info,[{math,sin,1},traced]),
    ?l {ok,[{reactivating_nodes,[ANode]}]}=inviso_tool:get_activities(),
    ?l timer:sleep(3600),
    %% Now the history shall have been redone including the new inviso command.
    ?l ok=poll(rpc,call,[ANode,erlang,trace_info,[{math,sin,1},traced]],{traced,local},10),
    ?l {flags,[call]}=rpc:call(ANode,erlang,trace_info,[APid,flags]),
    ?l {ok,[]}=inviso_tool:get_activities(),

    %% Check the get_autostart function. We know that we use the standard options
    %% generator and the tracecases activated above.
    ?l {ok,{AutostartData1,NodeResults3}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l true=check_noderesults(Nodes,
			      fun({_N,{ok,{[{dependency,infinity}],{tdg,{_M,_F,TDlist}}}}})
				 when list(TDlist)->
				      true;
				 (_) ->
				      false
			      end,
			      NodeResults3),
    %% Check the tracecase that shall be activated and their order.
    ?l TraceCaseFileNameInit=filename:join(DataDir,"./tracecase_init.trc"),
    ?l TraceCaseFileName1=filename:join(DataDir,"./tracecase1_on.trc"),
    ?l TraceCaseFileName2=filename:join(DataDir,"./tracecase2_on.trc"),
    ?l [{file,{TraceCaseFileNameInit,[]}},
	{file,{TraceCaseFileName1,[{'ProcessName',inviso_tool_test_proc}]}},
	{mfa,{inviso,tpl,[math,module_info,0,[]]}},
	{file,{TraceCaseFileName2,[]}},
	{mfa,{inviso,tpl,[math,sin,1,[]]}}]=AutostartData1,

    %% Try to activate a faulty tracecase. We shall get the same history as before.
    ?l ok=inviso_tool:atc(tracecase3,id3,[]),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    ?l {ok,{AutostartData1,NodeResults3}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),

    %% Now deactivate a trace case.
    ?l inviso_tool:dtc(tracecase1,id1),
    %% Check that the now deactivated trace case is not part of autostart data
    %% if requested from the tool.
    ?l {ok,{AutostartData2,_NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l [{file,{TraceCaseFileNameInit,[]}},
	{mfa,{inviso,tpl,[math,module_info,0,[]]}},
	{file,{TraceCaseFileName2,[]}},
	{mfa,{inviso,tpl,[math,sin,1,[]]}}]=AutostartData2,
    %% Now tracing shall be removed since we deactivated tracecase1.
    ?l lists:foreach(fun(P)->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[P,flags]],
					{flags,[]},
					10),
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,1},traced]],
					{traced,false},
					10)
		     end,
		     TestProcs),

    %% Suspend the ANode again and check that when it is reactivated that
    %% tracecase1 is not redone at all. We use an ets table with a counter that is
    %% incremented every time the tracecase1 is executed.
    ?l {ok,[{ANode,ok}]}=rpc:call(CNode,inviso,suspend,[[ANode],testagain]),
    ?l [{counter,SideEffectCounter1}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l true=(SideEffectCounter1>0),
    ?l ok=inviso_tool:reactivate(ANode),
    ?l timer:sleep(3000),                    % The delay in tracecase2.
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    %% Now the reactivation is done, check that tracecase1 was not redone at ANode.
    ?l [{counter,SideEffectCounter1}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),

    %% Deactivate tracecase2.
    ?l another_return_value=inviso_tool:sync_dtc(tracecase2,id2),
    %% Immediately check the autostart data (again!). This time we want to see
    %% that the two inviso commands have been joined since there is no tracecase
    %% in between.
    ?l {ok,{AutostartData3,NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l [{file,{TraceCaseFileNameInit,[]}},
	{mfa,{inviso,tpl,[math,module_info,0,[]]}},
	{mfa,{inviso,tpl,[math,sin,1,[]]}}]=AutostartData3,

    %% Check that a deactivating tracecase is not redone at a reactivating node.
    ?l inviso_tool:sync_atc(tracecase5,id5,[]), % Updates the counter.
    %% Yet again suspend the node when we know that tracecase5 has been executed.
    ?l {ok,[{ANode,ok}]}=rpc:call(CNode,inviso,suspend,[[ANode],testagain2]),
    ?l timer:sleep(100),                     % Subscription reaches the tool.
    ?l [{counter,SideEffectCounter2A}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l [BNode|_]=OtherNodes,
    ?l [{counter,SideEffectCounter2B}]=rpc:call(BNode,ets,lookup,[test_proc_tab,counter]),
    ?l ok=inviso_tool:dtc(tracecase5,id5),   % In here there is a 2000 ms delay!
    ?l ok=inviso_tool:reactivate(ANode),
    %% Check that the reactivator is done, but that the tracecase remains. The
    %% reactivator should be done pretty quickly since there are no delays in the
    %% still active tracecases.
    ?l ok=poll(inviso_tool,
	       get_activities,
	       [],
	       {ok,[{tracecases,[{{tracecase5,id5},deactivating}]}]},
	       10),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},30),
    ?l [{counter,SideEffectCounter2A}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l SideEffectCounter2B1=SideEffectCounter2B+1,
    ?l [{counter,SideEffectCounter2B1}]=rpc:call(BNode,ets,lookup,[test_proc_tab,counter]),

    %% Check the flush function. It is difficult to find out if it really flushed.
    ?l {ok,NodeResults4}=inviso_tool:flush(),
    ?l true=check_noderesults(Nodes,ok,NodeResults4),

    %% Check that this function still has a trace pattern. We are going to stop session
    %% and check that it is still there.
    ?l lists:foreach(fun(N)->
			     ok=poll(rpc,
				     call,
				     [N,
				      erlang,
				      trace_info,
				      [{math,sin,1},traced]],
				     {traced,local},
				     20)
		     end,
		     Nodes),

    ?l stop_inviso_tool_session(CNode,1,Nodes),
    ?l {ok,{not_tracing,1,TDGargs}}=inviso_tool:get_session_data(),

    ?l {ok,NodeResults5}=inviso_tool:flush(Nodes),
    ?l true=check_noderesults(Nodes,fun({_,{error,_}})->true;(_)->false end,NodeResults5),
    ?l {ok,[]}=inviso_tool:flush(),

    %% Check that you can not start trace cases when the session is stopped.
    ?l {error,no_session}=inviso_tool:atc(tracecase2,id3,[]),
    %% But the history shall be there to retrieve.
    ?l {ok,{AutostartData3,NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l {ok,{inactive,running}}=inviso_tool:get_node_status(ANode),

    %% Check that the trace pattern is still there.
    ?l lists:foreach(fun(N)->
			     ok=poll(rpc,
				     call,
				     [N,
				      erlang,
				      trace_info,
				      [{math,sin,1},traced]],
				     {traced,local},
				     20)
		     end,
		     Nodes),

    %% Now start a session and check that the trace patterns is gone.
    ?l start_inviso_tool_session(CNode,[],2,Nodes),
    ?l lists:foreach(fun(N)->
			     ok=poll(rpc,
				     call,
				     [N,
				      erlang,
				      trace_info,
				      [{math,sin,1},traced]],
				     {traced,false},
				     20)
		     end,
		     Nodes),
    ?l stop_inviso_tool_session(CNode,2,Nodes),

    ?l stop_inviso_tool(CNode,Nodes),
    ok.
%% -----------------------------------------------------------------------------

%% This test case tests the rtc trace case mechanism.
dist_rtc(doc) -> [""];
dist_rtc(suite) -> [];
dist_rtc(Config) when is_list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=node(),
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=filename:join(?config(priv_dir,Config),""),
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l start_inviso_tool(Nodes,CNode,Opts),
    ?l start_inviso_tool_session(CNode,[],1,Nodes),
    %% Check that the initial tracecase has been executed at all tracing nodes.
    ?l lists:foreach(fun(N)->
			     ok=poll(rpc,
				     call,
				     [N,
				      erlang,
				      trace_info,
				      [{lists,module_info,1},traced]],
				     {traced,local},
				     20)
		     end,
		     Nodes),
    %% Start a test process at every node with a runtime component.
    ?l lists:foreach(fun(N)->spawn(N,?MODULE,test_proc_init,[]) end,Nodes),

    %% Let the processes start
    timer:sleep(100),

    %% Find the pids of the test processes.
    ?l TestProcs=lists:map(fun(N)->rpc:call(N,erlang,whereis,[inviso_tool_test_proc]) end,
			   Nodes),
    ?l true=(1=<length(TestProcs)),
    ?l [ANode|_]=Nodes,
    ?l [{counter,Val}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    %% Call the tracecase as an rtc.
    ?l inviso_tool:sync_rtc(tracecase5,[]), % Updates the counter.
    ?l [{counter,Val2}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l true=(Val2==Val+1),
    ?l inviso_tool:sync_rtc(tracecase5,[]), % Updates the counter.
    ?l [{counter,Val3}]=rpc:call(ANode,ets,lookup,[test_proc_tab,counter]),
    ?l true=(Val3==Val2+1),

    %% Now we stop the session and restore it again.
    ?l stop_inviso_tool_session(CNode,1,Nodes),
    ?l {ok,{2,_InvisoReturn}}=inviso_tool:restore_session(),
    %% The tracecase shall be done twice then.
    ?l ok=poll(rpc,call,[ANode,ets,lookup,[test_proc_tab,counter]],
	       fun([{counter,V}]) when V==Val3+2 -> true;
		  (_) -> false
	       end,
	       20),
    ?l stop_inviso_tool_session(CNode,2,Nodes),

    ?l {ok,{AutostartData,_NodeResults}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l [{file,{_FileNameInit,_}},{file,{FileName,Bindings}},{file,{FileName,Bindings}}]=
	AutostartData,
    ?l stop_inviso_tool(CNode,Nodes),
    ok.
%% -----------------------------------------------------------------------------


%% This test case tests mainly that reconnect and reinitiations of a node works.
dist_reconnect(doc) -> [""];
dist_reconnect(suite) -> [];
dist_reconnect(Config) when list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|OtherNodes]=RemoteNodes,
    CNode=node(),
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=filename:join(?config(priv_dir,Config),""),
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l start_inviso_tool(Nodes,CNode,Opts), 
    ?l start_inviso_tool_session(CNode,[],1,Nodes),
    %% Start a test process at every node with a runtime component.
    ?l lists:foreach(fun(N)->spawn(N,?MODULE,test_proc_init,[]) end,Nodes),

    %% Let the processes start
    timer:sleep(100),
    
    %% Find the pids of the test processes.
    ?l TestProcs=lists:map(fun(N)->rpc:call(N,erlang,whereis,[inviso_tool_test_proc]) end,
			   Nodes),
    ?l true=(1=<length(TestProcs)),
    ?l ok=inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},20),

    %% Now we want to crash a node. Lets choose the RegExp node, then we can test
    %% that regexp expansion is done elsewhere too.
    ?l test_server:stop_node(RegExpNode),
    ?l ok=poll(net_adm,ping,[RegExpNode],pang,10),
    %% Make time for the monitoring signal to reach inviso_c and for inviso_c to
    %% inform its subscribers e.g inviso_tool.
    ?l timer:sleep(100),
    ?l {_,[{_Node,Modules}]}=
	inviso_tool_lib:expand_module_names([node()],"application.*",[]),
    ?l NrOfModules=length(Modules),
    %% This also checks that regexp expansion can be made at another node
    %% than RexExpNode.
    ?l {ok,NodeResults1}=inviso_tool:inviso(tp,["application.*",module_info,0,[]]),
    ?l true=check_noderesults(OtherNodes,
			      fun({_N,{ok,Ints}}) when list(Ints) ->
				      NrOfModules=lists:sum(Ints),
				      true;
				 (_) ->
				      false
			      end,
			      NodeResults1),

    %% Do some faulty tests on the dead node.
    ?l {ok,{ok,[{RegExpNode,{error,down}}]}}=
	inviso_tool:reinitiate_session([RegExpNode]),
    ?l {ok,down}=inviso_tool:get_node_status(RegExpNode),

    %% Now it is time to restart the crashed node and reconnect it and then
    %% finally reinitiate it.
    ?l RegExpNodeString=atom_to_list(RegExpNode),
    ?l {match,Pos,1}=regexp:first_match(RegExpNodeString,"@"),
    ?l RegExpNodeName=list_to_atom(lists:sublist(RegExpNodeString,Pos-1)),
    ?l test_server:start_node(RegExpNodeName,peer,[]),
    ?l ok=poll(net_adm,ping,[RegExpNode],pong,20),
    ?l SuiteDir=filename:dirname(code:which(?MODULE)),
    ?l true=rpc:call(RegExpNode,code,add_patha,[SuiteDir]),
    ?l ok=rpc:call(RegExpNode,application,start,[runtime_tools]),
    ?l ok=poll(rpc,call,[RegExpNode,erlang,whereis,[inviso_rt]],undefined,20),
    ?l {ok,down}=inviso_tool:get_node_status(RegExpNode),

    %% Restart the test process.
    ?l spawn(RegExpNode,?MODULE,test_proc_init,[]),
    ?l ok=poll(rpc,
	       call,
	       [RegExpNode,erlang,whereis,[inviso_tool_test_proc]],
	       fun(P) when pid(P) -> true;
		  (undefined) -> false
	       end,
	       10),
    ?l TPid=rpc:call(RegExpNode,erlang,whereis,[inviso_tool_test_proc]),
    ?l {ok,[{RegExpNode,{ok,{inactive,running}}}]}=inviso_tool:reconnect_nodes([RegExpNode]),
    %% Try to reconnect the node again and an unknown node.
    ?l UnknownNode='unknown@nonexistant',
    ?l {ok,[{RegExpNode,{error,already_connected}},{UnknownNode,{error,unknown_node}}]}=
	inviso_tool:reconnect_nodes([RegExpNode,UnknownNode]),
    ?l {ok,{ok,[{RegExpNode,{ok,_}}]}}=inviso_tool:reinitiate_session([RegExpNode]),
    ?l ok=poll(rpc,
	       call,
	       [RegExpNode,erlang,trace_info,[TPid,flags]],
	       {flags,[call]},
	       10),
    ?l {ok,{ok,[{RegExpNode,{error,already_in_session}},{UnknownNode,{error,unknown_node}}]}}=
	inviso_tool:reinitiate_session([RegExpNode,UnknownNode]),

    %% Suspend RegExpNode and test that it can not be reinitiated.
    ?l {ok,[{RegExpNode,ok}]}=rpc:call(CNode,inviso,suspend,[[RegExpNode],yetatest]),
    ?l {ok,[{RegExpNode,{error,already_connected}}]}=inviso_tool:reconnect_nodes([RegExpNode]),
    ?l {ok,{ok,[{RegExpNode,{error,suspended}}]}}=inviso_tool:reinitiate_session([RegExpNode]),

    %% Now we start a tracecase that will never terminate. We then reactivate the
    %% suspended node. Then there will be a running reactivator and a running
    %% tracecase to kill.
    ?l ok=inviso_tool:atc(tracecase4,id4,[]), % This one will not terminate.
    ?l ok=inviso_tool:reactivate(RegExpNode),
    ?l ok=poll(inviso_tool,
	       get_activities,
	       [],
	       fun({ok,L}) when length(L)==2 -> true;
		  (_) -> false
	       end,
	       20),
    %% Now the reactivator and the tracecase shall be stuck(!)
    ?l {links,ToolLinks}=process_info(whereis(inviso_tool),links),
    ?l [P1,P2]=lists:foldl(fun(P,Acc)->case process_info(P,initial_call) of
					   {initial_call,{inviso_tool,_,_}} ->
					       [P|Acc];
					   _ ->
					       Acc
				       end
			   end,
			   [],
			   ToolLinks),
    ?l stop_inviso_tool_session(CNode,1,Nodes),
    %% Check that the processes are killed.
    ?l ok=poll(erlang,process_info,[P1],undefined,10),
    ?l ok=poll(erlang,process_info,[P2],undefined,10),
    ?l stop_inviso_tool(CNode,Nodes),
    ok.
%% -----------------------------------------------------------------------------

%% This test tests that we can adopt a running inviso runtime component and
%% mark it as tracing-running.
dist_adopt(doc) -> [""];
dist_adopt(suite) -> [];
dist_adopt(Config) when list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=node(),
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=filename:join(?config(priv_dir,Config),""),

    %% Then first start runtime components at different nodes for us to
    %% later adopt.
    ?l {ok,_IPid}=inviso:start(),
    ?l {ok,NodeResults1}=inviso:add_nodes(Nodes,a_tag,[{dependency,infinity}]),
    ?l true=check_noderesults(Nodes,{ok,new},NodeResults1),
    ?l [ANode|OtherNodes]=Nodes,
    ?l {ok,[{ANode,_LogResult}]}=
	inviso:init_tracing([ANode],
			    [{trace,{file,filename:join(PrivDir,"dist_adopt_adoptednode.log")}},
			     {ti,{file,filename:join(PrivDir,"dist_adopt_adoptednode.ti")}}]),
    ?l inviso:stop(),
    ?l ok=poll(erlang,whereis,[inviso_c],undefined,10),
    ?l lists:foreach(fun(N)->true=(is_pid(rpc:call(N,erlang,whereis,[inviso_rt]))) end,
		     Nodes),

    %% Now start the tool and watch it adopt the runtimes.
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l Options=[{nodes,Nodes},{c_node,CNode}|Opts],
    ?l {ok,_Pid}=inviso_tool:start(Options),
    ?l ok=poll(erlang,whereis,[inviso_tool],fun(X)->true=is_pid(X) end,10),
    ?l io:format("LoopData:~p~n",[inviso_tool:get_loopdata()]),
    ?l {ok,{1,InvisoReturn}}=inviso_tool:start_session([]),
    ?l io:format("Invisoreturn:~p~n",[InvisoReturn]),
    %% Now check that all nodes are tracing.
    ?l lists:foreach(fun(N)->ok=poll(rpc,
				     call,
				     [CNode,inviso,get_status,[[N]]],
				     fun({ok,[{_N,{ok,{tracing,running}}}]})->true;
					(_) ->false
				     end,
				     10)
		     end,
		     Nodes),

    %% At this point all nodes shall be tracing. However the initial tracecase
    %% shall not have been executed at ANode since it was adopted by the tool.
    ?l {traced,false}=rpc:call(ANode,erlang,trace_info,[{lists,module_info,1},traced]),
    ?l lists:foreach(fun(N)->
			     {traced,local}=
				 rpc:call(N,erlang,trace_info,[{lists,module_info,1},traced])
		     end,
		     OtherNodes),
    ?l stop_inviso_tool_session(CNode,1,Nodes),
    ?l [BNode|_]=OtherNodes,
    %% Since nodes are not cleared the pattern still be there.
    ?l {traced,local}=rpc:call(BNode,erlang,trace_info,[{lists,module_info,1},traced]),
    ?l start_inviso_tool_session(CNode,[],2,Nodes),
    ?l stop_inviso_tool_session(CNode,2,Nodes),
    ?l {ok,_NodeResults}=inviso_tool:stop(),
    ?l ok=poll(erlang,whereis,[inviso_tool],undefined,10),
    ?l ok=poll(rpc,call,[CNode,erlang,whereis,[inviso_c]],undefined,10),

    ok.
%% -----------------------------------------------------------------------------

%% This test tests that saving and restoring a history works.
dist_history(doc) -> [""];
dist_history(suite) -> [];
dist_history(Config) when list(Config) ->		     
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=RegExpNode,                       % We use a remote control component.
    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=filename:join(?config(priv_dir,Config),""),

    %% Start up the tool and a couple of inviso runtimes.
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l start_inviso_tool(Nodes,CNode,Opts), 
    ?l start_inviso_tool_session(CNode,[],1,Nodes),
    %% Start a test process at every node with a runtime component.
    ?l lists:foreach(fun(N)->spawn(N,?MODULE,test_proc_init,[]) end,Nodes),

    %% Activate tracing of the test process.
    ?l ok=inviso_tool:atc(tracecase1,id1,[{'ProcessName',inviso_tool_test_proc}]),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    ?l TestProcs=lists:map(fun(N)->rpc:call(N,erlang,whereis,[inviso_tool_test_proc]) end,
			   Nodes),
    ?l lists:foreach(fun(P)->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[P,flags]],
					{flags,[call]},
					10),
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,1},traced]],
					{traced,local},
					10)
		     end,
		     TestProcs),

    %% Create a history file.
    ?l AbsFileName=filename:join(PrivDir,"dist_history.his"),
    ?l {ok,AbsFileName}=inviso_tool:save_history(AbsFileName),
    ?l {ok,_FileInfo}=file:read_file_info(AbsFileName),

    %% Stop the tracing of the test process.
    ?l inviso_tool:sync_dtc(tracecase1,id1),
    ?l lists:foreach(fun(P)->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[P,flags]],
					{flags,[]},
					10),
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,1},traced]],
					{traced,false},
					10)
		     end,
		     TestProcs),
    %% Now stop the session.
    ?l stop_inviso_tool_session(CNode,1,Nodes),
    %% Restart the session using the previously saved history.
    ?l {ok,{2,_InvisoReturn}}=inviso_tool:restore_session(AbsFileName),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    %% Check that the history has been redone.
    ?l lists:foreach(fun(P)->
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[P,flags]],
					{flags,[call]},
					10),
			     ?l ok=poll(rpc,
					call,
					[node(P),erlang,trace_info,[{math,module_info,1},traced]],
					{traced,local},
					10)
		     end,
		     TestProcs),
    ?l {ok,_NodeResults1}=inviso_tool:inviso(tpl,[math,module_info,0,[]]),
    %% Also check that the restored history now is our history.
    ?l {ok,{AutostartData,_NodeResults2}}=
	inviso_tool:get_autostart_data(Nodes,{dependency,infinity}),
    ?l FNameInit=filename:join(DataDir,"tracecase_init.trc"),
    ?l FName1=filename:join(DataDir,"tracecase1_on.trc"),
    ?l [{file,{FNameInit,[]}},
	{file,{FName1,[{'ProcessName',inviso_tool_test_proc}]}},
	{mfa,{inviso,tpl,[math,module_info,0,[]]}}]=AutostartData,
    ?l stop_inviso_tool_session(CNode,2,Nodes),
    ?l NodeCounters=lists:foldl(fun(N,Acc)->[{_,X}]=rpc:call(N,ets,lookup,[test_proc_tab,counter]),
					    [{N,X}|Acc]
				end,
				[],
				Nodes),
    %% Remove the patterns set by the initial tracecase.
    ?l lists:foreach(fun(N)->rpc:call(N,
				      erlang,
				      trace_pattern,
				      [{lists,module_info,1},false,[local]])
		     end,
		     Nodes),
    %% Now we want to test that we can do restore on the current session.
    ?l {ok,{3,_InvisoReturn2}}=inviso_tool:restore_session(),
    ?l ok=poll(inviso_tool,get_activities,[],{ok,[]},10),
    %% Check that the history has been redone yet again.
    ?l lists:foreach(fun({N,X})->
			     [{counter,Y}]=
				 rpc:call(N,ets,lookup,[test_proc_tab,counter]),
			     Y=X+1
		     end,
		     NodeCounters),
    ?l lists:foreach(fun(N)->
			     {traced,local}=
				 rpc:call(N,erlang,trace_info,[{lists,module_info,1},traced])
		     end,
		     Nodes),

    ?l {error,session_active}=inviso_tool:reset_nodes(Nodes),
    %% Now stop the session and check that we can clear the nodes.
    ?l stop_inviso_tool_session(CNode,3,Nodes),
    ?l lists:foreach(fun(N)->{traced,local}=
				 rpc:call(N,erlang,trace_info,[{lists,module_info,1},traced])
		     end,
		     Nodes),
    ?l {ok,NodeResults3}=inviso_tool:reset_nodes(Nodes),
    ?l true=check_noderesults(Nodes,{ok,{new,running}},NodeResults3),
    ?l lists:foreach(fun(N)->{traced,false}=
				 rpc:call(N,erlang,trace_info,[{lists,module_info,1},traced])
		     end,
		     Nodes),
    ?l stop_inviso_tool(CNode,Nodes),

    %% Now we want to test that restoring a session at no active nodes will
    %% not result in a crash. (Previous error).
    ?l FaultyNodes=[gurka@nonexistant,tomat@nonexistant],
    ?l Options=[{nodes,FaultyNodes},{c_node,CNode}|Opts],
    ?l {ok,_Pid}=inviso_tool:start(Options),
    ?l ok=poll(erlang,whereis,[inviso_tool],fun(X)->true=is_pid(X) end,10),
    %% Now try to restore a session.
    ?l {ok,{_,{ok,[]}}}=inviso_tool:restore_session(AbsFileName),
    ?l {ok,down}=inviso_tool:get_node_status(gurka@nonexistant),
    %% Now stop the (useless) session.
    ?l {ok,{_,[]}}=inviso_tool:stop_session(),
    ?l stop_inviso_tool(CNode,[]),
    ok.
%% -----------------------------------------------------------------------------

%% This test tests a few strange situations when activating a session and there
%% are no nodes that can be initiated or reinitiated.
dist_start_session_special(doc) -> [""];
dist_start_session_special(suite) -> [];
dist_start_session_special(Config) when list(Config) ->
    RemoteNodes=get_remotenodes_config(Config),
    [RegExpNode|_]=RemoteNodes,
    CNode=RegExpNode,                       % We use a remote control component.
%    Nodes=RemoteNodes,
    DataDir=?config(data_dir,Config),
    PrivDir=filename:join(?config(priv_dir,Config),""),

    %% Start up the tool but with no exiting nodes.
    FaultyNodes=[gurka@nonexistant,tomat@nonexistant],
    Opts=[{regexp_node,RegExpNode},
	  {tdg,{?MODULE,tdg,[PrivDir]}},
	  {tc_def_file,?TC_DEF_FILE},
	  {initial_tcs,[{tracecase_init,[]}]},
	  {dir,DataDir}],                    % This is where we find tracecases.
    ?l Options=[{nodes,FaultyNodes},{c_node,CNode}|Opts],
    ?l {ok,_Pid}=inviso_tool:start(Options),
    ?l ok=poll(erlang,whereis,[inviso_tool],fun(X)->true=is_pid(X) end,10),
    %% Now try to initate a session.
    ?l {ok,{SessionNr,{ok,[]}}}=inviso_tool:start_session(),
    ?l {ok,down}=inviso_tool:get_node_status(gurka@nonexistant),
    %% Now stop the (useless) session.
    ?l {ok,{SessionNr,[]}}=inviso_tool:stop_session(),

    %% Now start again, still no useful nodes.
    ?l {ok,{SessionNr2,{ok,[]}}}=inviso_tool:start_session(),
    ?l {ok,{SessionNr2,[]}}=inviso_tool:stop_session(),
    ?l stop_inviso_tool(CNode,[]), % No nodes are connected.

    ok.
%% -----------------------------------------------------------------------------

    
%% ==============================================================================
%% Help functions.
%% ==============================================================================
    
%% Help function starting the inviso_tool with runtime components at Nodes
%% and the inviso control component at CNode. OtherOpts shall contain all other
%% necessary options except nodes and c_node. Returns nothing significant.
start_inviso_tool(Nodes,CNode,OtherOpts) -> 
    ?l Options=[{nodes,Nodes},{c_node,CNode}|OtherOpts],
    ?l {ok,_Pid}=inviso_tool:start(Options),
    ?l ok=poll(erlang,whereis,[inviso_tool],fun(X)->true=is_pid(X) end,10),
    %% Now the runtime components shall be started but no tracing started.
    ?l lists:foreach(fun(N)->ok=poll(rpc,
				     call,
				     [CNode,inviso,get_status,[[N]]],
				     fun({ok,[{_N,{ok,{new,running}}}]})->true;
					(_) ->false
				     end,
				     10)
		     end,
		     Nodes),
    true.
%% -----------------------------------------------------------------------------

%% Stops the inviso_tool.
stop_inviso_tool(CNode,Nodes) ->
    ?l {ok,NodeResults}=inviso_tool:stop(),
    ?l true=check_noderesults(Nodes,ok,NodeResults),
    ?l ok=poll(erlang,whereis,[inviso_tool],undefined,10),
    %% Check that all inviso components are gone.
    ?l ok=poll(rpc,call,[CNode,erlang,whereis,[inviso_c]],undefined,10),
    ?l lists:foreach(fun(N)->ok=poll(rpc,
				     call,
				     [N,erlang,whereis,[inviso_rt]],
				     undefined,
				     10)
		     end,
		     Nodes),
    true.
%% -----------------------------------------------------------------------------

%% Starts a trace session. Returns the InvisoReturn part of the return value.
start_inviso_tool_session(CNode,MoreTDGargs,SessionNr,Nodes) ->
    ?l {ok,{SessionNr,InvisoReturn}}=inviso_tool:start_session(MoreTDGargs),
    %% Now check that all nodes are tracing.
    ?l lists:foreach(fun(N)->ok=poll(rpc,
				     call,
				     [CNode,inviso,get_status,[[N]]],
				     fun({ok,[{_N,{ok,{tracing,running}}}]})->true;
					(_) ->false
				     end,
				     10),
			     %% Check that the initial trace case is executed.
			     ?l ok=poll(rpc,
					call,
					[N,erlang,trace_info,[{lists,module_info,1},traced]],
					{traced,local},
					10)
		     end,
		     Nodes),
    InvisoReturn.
%% -----------------------------------------------------------------------------

%% Stops a trace session.
stop_inviso_tool_session(CNode,SessionNr,Nodes) ->
    ?l {ok,{SessionNr,NodeResults}}=inviso_tool:stop_session(),
    ?l true=check_noderesults(Nodes,ok,NodeResults),
    %% Now the runtimes shall not be tracing any longer.
    ?l lists:foreach(fun(N)->ok=poll(rpc,
				     call,
				     [CNode,inviso,get_status,[[N]]],
				     fun({ok,[{_N,{ok,{idle,running}}}]})->true;
					(_) ->false
				     end,
				     10)
		     end,
		     Nodes),
    true.
%% -----------------------------------------------------------------------------

%% Help function checking that there is a Result for each node in Nodes.
%% Returns 'true' if successful.
check_noderesults(Nodes,Fun,[{Node,Result}|Rest]) when function(Fun) ->
    case Fun({Node,Result}) of
	true ->
	    case lists:member(Node,Nodes) of
		true ->
		    check_noderesults(lists:delete(Node,Nodes),Fun,Rest);
		false ->                     % Not good.
		    unknown_node_in_returnvalue
	    end;
	_ ->
	    illegal_result
    end;
check_noderesults(Nodes,Result,[{Node,Result}|Rest]) ->
    case lists:member(Node,Nodes) of
	true ->
	    check_noderesults(lists:delete(Node,Nodes),Result,Rest);
	false ->                             % Not good.
	    unknown_node_in_returnvalue
    end;
check_noderesults([],_,[]) ->
    true;
check_noderesults(X,Y,Z) ->
    io:format("Bad arguments to check noderesults:~w~n~w~n~w~n",[X,Y,Z]),
    false.
%% ------------------------------------------------------------------------------

%% Help function which waits for a function call to become Result. This is useful
%% if what we are waiting for can happend independantly of indications we have
%% access to.
poll(_,_,_,_,0) ->
    error;
poll(M,F,Args,Result,Times) ->
    try apply(M,F,Args) of
	What when function(Result) ->
	    case Result(What) of
		true ->
		    ok;
		X ->
		    io:format("Poll: ~w:~w ~w ~w ~w~n",[M,F,Args,Result,X]),
		    timer:sleep(100),
		    poll(M,F,Args,Result,Times-1)
	    end;
	Result ->
	    ok;
	X ->
	    io:format("Poll: ~w:~w ~w ~w ~w~n",[M,F,Args,Result,X]),
	    timer:sleep(100),
	    poll(M,F,Args,Result,Times-1)
    catch
	error:Reason ->
	    io:format("Apply in suite-function poll/5 failed, ~w~n",[Reason]),
	    timer:sleep(100),
	    poll(M,F,Args,Result,Times-1)
    end.
%% ------------------------------------------------------------------------------

%% ------------------------------------------------------------------------------
%% The Tracer Data Generator function.
%% ------------------------------------------------------------------------------

-define(I2L(Arg),integer_to_list(Arg)).

tdg(Node,{{Y,Mo,D},{H,Mi,S}},PrivDir) ->
    NameStr=atom_to_list(Node)++"_"++?I2L(Y)++"-"++?I2L(Mo)++"-"++?I2L(D)++"_"++
	?I2L(H)++"-"++?I2L(Mi)++"-"++?I2L(S),
    LogTD={file,filename:join(PrivDir,NameStr++".log")},
    TiTD={file,filename:join(PrivDir,NameStr++".ti")},
    [{trace,LogTD},{ti,TiTD}].
%% ------------------------------------------------------------------------------


%% ------------------------------------------------------------------------------
%% Handling the test server Config.
%% ------------------------------------------------------------------------------

insert_remotenode_config(Name,Node,Config) ->
    [{remotenode,{Name,Node}}|Config].
%% ------------------------------------------------------------------------------

insert_timetraphandle_config(Handle,Config) ->
    [{timetraphandle,Handle}|Config].
%% ------------------------------------------------------------------------------

get_remotenode_config(Name, [{remotenode, {Name, Node}}| _Cs]) ->
    Node;
get_remotenode_config(Name, [_C | Cs]) ->
    get_remotenode_config(Name, Cs);
get_remotenode_config(Name, []) ->
    exit({no_remotenode, Name}).

%% ------------------------------------------------------------------------------

get_timetraphandle_config(Config) ->
    {value,{_,Handle}}=lists:keysearch(timetraphandle,1,Config),
    Handle.
%% ------------------------------------------------------------------------------

get_remotenodes_config([{remotenode,{_Name,Node}}|Config]) ->
    [Node|get_remotenodes_config(Config)];
get_remotenodes_config([_|Config]) ->
    get_remotenodes_config(Config);
get_remotenodes_config([]) ->
    [].
%% ------------------------------------------------------------------------------

remove_remotenode_config(Name, [{remotenode, {Name, _}} | Cs]) ->
    Cs;
remove_remotenode_config(Name, [C | Cs]) ->
    [C | remove_remotenode_config(Name, Cs)];
remove_remotenode_config(_Name, []) ->
    [].
%% ------------------------------------------------------------------------------

remove_timetraphandle_config(Config) ->
    lists:keydelete(timetraphandle,1,Config).
%% ------------------------------------------------------------------------------


%% ==============================================================================
%% Code for a test process which can be started.
%% ==============================================================================

%% The test proc is also responsible for owning a side effect table. The table
%% can be updated by tracecases.
test_proc_init() ->
    register(inviso_tool_test_proc,self()),
    ets:new(test_proc_tab,[named_table,public]),
    ets:insert(test_proc_tab,{counter,0}),
    test_proc_loop().

test_proc_loop() ->
    receive
	{apply,M,F,Args} ->
	    apply(M,F,Args),
	    test_proc_loop();
	X ->
	    io:format("Got ~w~n",[X]),
	    test_proc_loop()
    end.
%% ------------------------------------------------------------------------------