Improve eval and node_start and add new options for ct_slave

4 files changed, 162 insertions, 47 deletions

diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl
index 94afccde79..b14e7527e5 100644
--- a/lib/common_test/src/ct_master.erl
+++ b/lib/common_test/src/ct_master.erl
@@ -28,7 +28,7 @@
 
 -export([abort/0,abort/1,progress/0]).
 
--export([init_master/6, init_node_ctrl/3]).
+-export([init_master/7, init_node_ctrl/3]).
 
 -export([status/2]).
 
@@ -101,11 +101,12 @@ run([TS|TestSpecs],AllowUserTerms,InclNodes,ExclNodes) when is_list(TS),
 	    TSRec=#testspec{logdir=AllLogDirs,
 			    config=StdCfgFiles,
 			    userconfig=UserCfgFiles,
+			    node_start=AllNSOpts,
 			    event_handler=AllEvHs} ->
 	        AllCfgFiles = {StdCfgFiles, UserCfgFiles},
 		RunSkipPerNode = ct_testspec:prepare_tests(TSRec),
 		RunSkipPerNode2 = exclude_nodes(ExclNodes,RunSkipPerNode),
-		run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs,[],[],TS1)
+		run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs,[],[],AllNSOpts,TS1)
 	end,
     [{TS,Result} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)];
 run([],_,_,_) ->
@@ -161,11 +162,12 @@ run_on_node([TS|TestSpecs],AllowUserTerms,Node) when is_list(TS),is_atom(Node) -
 		{error,Reason};
 	    TSRec=#testspec{logdir=AllLogDirs,
 			    config=StdCfgFiles,
+			    node_start=AllNSOpts,
 			    userconfig=UserCfgFiles,
 			    event_handler=AllEvHs} ->
 	        AllCfgFiles = {StdCfgFiles, UserCfgFiles},
 		{Run,Skip} = ct_testspec:prepare_tests(TSRec,Node),
-		run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs,[],[],TS1)
+		run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs,[],[],AllNSOpts,TS1)
 	end,
     [{TS,Result} | run_on_node(TestSpecs,AllowUserTerms,Node)];
 run_on_node([],_,_) ->
@@ -187,7 +189,7 @@ run_on_node(TestSpecs,Node) ->
 
 run_all([{Node,Run,Skip}|Rest],AllLogDirs,
 	{AllStdCfgFiles, AllUserCfgFiles}=AllCfgFiles,
-	AllEvHs,NodeOpts,LogDirs,Specs) ->
+	AllEvHs,NodeOpts,LogDirs,NSOptions,Specs) ->
     LogDir =
 	lists:foldl(fun({N,Dir},_Found) when N == Node ->
 			    Dir;
@@ -214,19 +216,20 @@ run_all([{Node,Run,Skip}|Rest],AllLogDirs,
 		       ({_N,_H,_A},Hs) -> Hs;
 		       ({H,A},Hs) -> [{H,A}|Hs]
 		    end,[],AllEvHs),
+
     NO = {Node,[{prepared_tests,{Run,Skip},Specs},
 		{logdir,LogDir},
 		{config,StdCfgFiles},
 		{event_handler,EvHs}] ++ UserCfgFiles},
-    run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,[NO|NodeOpts],[LogDir|LogDirs],Specs);
-run_all([],AllLogDirs,_,AllEvHs,NodeOpts,LogDirs,Specs) ->
+    run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,[NO|NodeOpts],[LogDir|LogDirs],NSOptions,Specs);
+run_all([],AllLogDirs,_,AllEvHs,NodeOpts,LogDirs,NSOptions,Specs) ->
     Handlers = [{H,A} || {Master,H,A} <- AllEvHs, Master == master],
     MasterLogDir = case lists:keysearch(master,1,AllLogDirs) of
 		       {value,{_,Dir}} -> Dir;
 		       false -> "."
 		   end,
     log(tty,"Master Logdir","~s",[MasterLogDir]),
-    start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,LogDirs,Specs),
+    start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,LogDirs,NSOptions,Specs),
     ok.
     
 
@@ -264,17 +267,17 @@ progress() ->
 %%% MASTER, runs on central controlling node.
 %%%-----------------------------------------------------------------
 start_master(NodeOptsList) ->
-    start_master(NodeOptsList,[],".",[],[]).
+    start_master(NodeOptsList,[],".",[],[],[]).
 
-start_master(NodeOptsList,EvHandlers,MasterLogDir,LogDirs,Specs) ->
+start_master(NodeOptsList,EvHandlers,MasterLogDir,LogDirs,NSOptions,Specs) ->
     Master = spawn_link(?MODULE,init_master,[self(),NodeOptsList,EvHandlers,
-					     MasterLogDir,LogDirs,Specs]),
+					     MasterLogDir,LogDirs,NSOptions,Specs]),
     receive 
 	{Master,Result} -> Result
     end.	    
 
 %%% @hidden
-init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,Specs) ->
+init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,NSOptions,Specs) ->
     case whereis(ct_master) of
 	undefined ->
 	    register(ct_master,self()),
@@ -327,9 +330,10 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,Specs) ->
 	Pid when is_pid(Pid) ->
 	    ok
     end,
-    init_master1(Parent,NodeOptsList,LogDirs).
+    init_master1(Parent,NodeOptsList,NSOptions,LogDirs).
 
-init_master1(Parent,NodeOptsList,LogDirs) ->
+init_master1(Parent,NodeOptsList,NSOptions,LogDirs) ->
+    start_nodes(NSOptions),
     {Inaccessible,NodeOptsList1} = ping_nodes(NodeOptsList,[],[]),
     case Inaccessible of
 	[] ->
@@ -344,7 +348,7 @@ init_master1(Parent,NodeOptsList,LogDirs) ->
 			"Proceeding without: ~p",[Inaccessible]),
 		    init_master2(Parent,NodeOptsList1,LogDirs);
 		"r\n" ->
-		    init_master1(Parent,NodeOptsList,LogDirs);
+		    init_master1(Parent,NodeOptsList,NSOptions,LogDirs);
 		_ ->
 		    log(html,"Aborting Tests","",[]),
 		    ct_master_event:stop(),
@@ -698,6 +702,23 @@ reply(Result,To) ->
     To ! {self(),Result},
     ok.
 
+start_nodes([])->
+    ok;
+start_nodes([{NodeName, Options}|Rest])->
+    [NodeS,HostS]=string:tokens(atom_to_list(NodeName), "@"),
+    Node=list_to_atom(NodeS),
+    Host=list_to_atom(HostS),
+    {value, {callback_module, Callback}, Options2}=
+	lists:keytake(callback_module, 1, Options),
+    io:format("Starting node ~p on host ~p with callback ~p...~n", [Node, Host, Callback]),
+    case Callback:start(Host, Node, Options2) of
+	{ok, NodeName} ->
+	    io:format("Node ~p started successfully~n", [NodeName]);
+	{error, Reason, _NodeName} ->
+	    io:format("Failed to start node ~p! Reason: ~p~n", [NodeName, Reason])
+    end,
+    start_nodes(Rest).
+
 %cast(Msg) ->
 %    cast(whereis(ct_master),Msg).
 
diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl
index bf7ee8863c..09d8820731 100644
--- a/lib/common_test/src/ct_slave.erl
+++ b/lib/common_test/src/ct_slave.erl
@@ -29,7 +29,7 @@
 
 -record(options, {username, password, boot_timeout, init_timeout,
 		  startup_timeout, startup_functions, monitor_master,
-		  kill_if_fail}).
+		  kill_if_fail, erl_flags}).
 
 %%%-----------------------------------------------------------------
 %%% @spec start(Node) -> Result
@@ -39,7 +39,8 @@
 %%%	       {error, started_not_connected, NodeName} |
 %%%	       {error, boot_timeout, NodeName} |
 %%%	       {error, init_timeout, NodeName} |
-%%%	       {error, startup_timeout, NodeName}
+%%%	       {error, startup_timeout, NodeName} |
+%%%	       {error, not_alive, NodeName}
 %%%   NodeName = atom()
 %%% @doc Starts an Erlang node with name <code>Node</code> on the local host.
 %%% @see start/3
@@ -55,7 +56,8 @@ start(Node)->
 %%%	       {error, started_not_connected, NodeName} |
 %%%	       {error, boot_timeout, NodeName} |
 %%%	       {error, init_timeout, NodeName} |
-%%%	       {error, startup_timeout, NodeName}
+%%%	       {error, startup_timeout, NodeName} |
+%%%	       {error, not_alive, NodeName}
 %%%   NodeName = atom()
 %%% @doc Starts an Erlang node with name <code>Node</code> on host
 %%% <code>Host</code> with the default options.
@@ -74,7 +76,8 @@ start(Host, Node)->
 %%%		  {startup_timeout, StartupTimeout} |
 %%%		  {startup_functions, StartupFunctions} |
 %%%		  {monitor_master, Monitor} |
-%%%		  {kill_if_fail, KillIfFail}
+%%%		  {kill_if_fail, KillIfFail} |
+%%%		  {erl_flags, ErlangFlags}
 %%%   Username = string()
 %%%   Password = string()
 %%%   BootTimeout = integer()
@@ -85,13 +88,15 @@ start(Host, Node)->
 %%%   Module = atom()
 %%%   Function = atom()
 %%%   Arguments = [term]
-%%%   Monitor = bool
-%%%   KillIfFail = bool
+%%%   Monitor = bool()
+%%%   KillIfFail = bool()
+%%%   ErlangFlags = string()
 %%%   Result = {ok, NodeName} | {error, already_started, NodeName} |
 %%%	       {error, started_not_connected, NodeName} |
 %%%	       {error, boot_timeout, NodeName} |
 %%%	       {error, init_timeout, NodeName} |
-%%%	       {error, startup_timeout, NodeName}
+%%%	       {error, startup_timeout, NodeName} |
+%%%	       {error, not_alive, NodeName}
 %%%   NodeName = atom()
 %%% @doc Starts an Erlang node with name <code>Node</code> on host
 %%% <code>Host</code> as specified by the combination of options in
@@ -129,27 +134,33 @@ start(Host, Node)->
 %%% </list></p>
 %%%
 %%% <p>Option <code>monitor_master</code> specifies, if the slave node should be
-%%% stopped in case of master node stop. Defaults to false</p>
+%%% stopped in case of master node stop. Defaults to true.</p>
 %%%
 %%% <p>Option <code>kill_if_fail</code> specifies, if the slave node should be
 %%% killed in case of a timeout during initialization or startup.
 %%% Defaults to true. Note that node also may be still alive it the boot
 %%% timeout occurred, but it will not be killed in this case.</p>
 %%%
+%%% <p>Option <code>erlang_flags</code> specifies, which flags will be added
+%%% to the parameters of the <code>erl</code> executable.</p>
+%%%
 %%% <p>Special return values are:
 %%% <list>
 %%%  <item><code>{error, already_started, NodeName}</code> - if the node with
 %%%   the given name is already started on a given host;</item>
 %%%  <item><code>{error, started_not_connected, NodeName}</code> - if node is
 %%%  started, but not connected to the master node.</item>
+%%%  <item><code>{error, not_alive, NodeName}</code> - if node on which the
+%%%   <code>ct_slave:start/3</code> is called, is not alive. Note that
+%%%   <code>NodeName</code> is the name of current node in this case.</item>
 %%% </list></p>
 %%%
 start(Host, Node, Options)->
     ENode = enodename(Host, Node),
-    case net_kernel:longnames() of
-	ignored->
-	    {error, not_alive};
-	_->
+    case erlang:is_alive() of
+	false->
+	    {error, not_alive, node()};
+	true->
 	    case is_started(ENode) of
 		false->
 		    OptionsRec = fetch_options(Options),
@@ -211,12 +222,13 @@ fetch_options(Options)->
     InitTimeout = get_option_value(init_timeout, Options, 1),
     StartupTimeout = get_option_value(startup_timeout, Options, 1),
     StartupFunctions = get_option_value(startup_functions, Options, []),
-    Monitor = get_option_value(monitor_master, Options, false),
+    Monitor = get_option_value(monitor_master, Options, true),
     KillIfFail = get_option_value(kill_if_fail, Options, true),
+    ErlFlags = get_option_value(erl_flags, Options, []),
     #options{username=UserName, password=Password,
 	     boot_timeout=BootTimeout, init_timeout=InitTimeout,
 	     startup_timeout=StartupTimeout, startup_functions=StartupFunctions,
-	     monitor_master=Monitor, kill_if_fail=KillIfFail}.
+	     monitor_master=Monitor, kill_if_fail=KillIfFail, erl_flags=ErlFlags}.
 
 % send a message when slave node is started
 % @hidden
@@ -282,7 +294,7 @@ do_start(Host, Node, Options)->
     MasterHost = gethostname(),
     if
 	MasterHost == Host ->
-	    spawn_local_node(Node);
+	    spawn_local_node(Node, Options);
 	true->
 	    spawn_remote_node(Host, Node, Options)
     end,
@@ -338,14 +350,15 @@ gethostname()->
     list_to_atom(Hostname).
 
 % get cmd for starting Erlang
-get_cmd(Node)->
+get_cmd(Node, Flags)->
     Cookie = erlang:get_cookie(),
     "erl -detached -noinput -setcookie "++ atom_to_list(Cookie) ++
-    long_or_short() ++ atom_to_list(Node).
+    long_or_short() ++ atom_to_list(Node) ++ " " ++ Flags.
 
 % spawn node locally
-spawn_local_node(Node)->
-    Cmd = get_cmd(Node),
+spawn_local_node(Node, Options)->
+    ErlFlags = Options#options.erl_flags,
+    Cmd = get_cmd(Node, ErlFlags),
     %io:format("Running cmd: ~p~n", [Cmd]),
     open_port({spawn, Cmd}, [stream]).
 
@@ -368,6 +381,7 @@ check_for_ssh_running()->
 spawn_remote_node(Host, Node, Options)->
     Username = Options#options.username,
     Password = Options#options.password,
+    ErlFlags = Options#options.erl_flags,
     SSHOptions = case {Username, Password} of
 	{[], []}->
 	    [];
@@ -379,7 +393,7 @@ spawn_remote_node(Host, Node, Options)->
     check_for_ssh_running(),
     {ok, SSHConnRef} = ssh:connect(atom_to_list(Host), 22, SSHOptions),
     {ok, SSHChannelId} = ssh_connection:session_channel(SSHConnRef, infinity),
-    ssh_connection:exec(SSHConnRef, SSHChannelId, get_cmd(Node), infinity).
+    ssh_connection:exec(SSHConnRef, SSHChannelId, get_cmd(Node, ErlFlags), infinity).
 
 % call functions on a remote Erlang node
 call_functions(_Node, [])->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index a0da079c54..db3efb16e8 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -270,33 +270,52 @@ collect_tests(Terms,TestSpec,Relaxed) ->
     put(relaxed,Relaxed),
     TestSpec1 = get_global(Terms,TestSpec),
     TestSpec2 = get_all_nodes(Terms,TestSpec1),
-    case catch evaluate(Terms,TestSpec2) of
+    % filter out node_start options and save them into the specification
+    {Terms2, TestSpec3} = filter_nodestart_specs(Terms, [], TestSpec2),
+    % only save the 'global' evals and evals for nodes which have no node_start
+    {Terms3, TestSpec4} = filter_evals(Terms2, [], TestSpec3),
+    % after evaluation, only valid terms exist in the specification list
+    Terms4 = case catch evaluate(Terms3, [], TestSpec4) of
 	{error,{Node,{M,F,A},Reason}} ->
 	    io:format("Error! Common Test failed to evaluate ~w:~w/~w on ~w. "
-		      "Reason: ~p~n~n", [M,F,A,Node,Reason]);
-	_ -> ok
+		      "Reason: ~p~n~n", [M,F,A,Node,Reason]),
+	    Terms3;
+	NewTerms -> NewTerms
     end,
-    add_tests(Terms,TestSpec2).
-    
-evaluate([{eval,NodeRef,{M,F,Args}}|Ts],Spec) ->
-    Node = ref2node(NodeRef,Spec#testspec.nodes),
+    add_tests(Terms4,TestSpec4).
+
+evaluate([{eval,Node,[{_,_,_}|_]=Mfas}|Ts],NewTerms,Spec)->
+    EvalTerms = lists:map(fun(Mfa)->
+	{eval, Node, Mfa}
+    end,
+    Mfas),
+    evaluate([EvalTerms|Ts], NewTerms, Spec);
+evaluate([{eval,[{_,_,_}|_]=Mfas}|Ts],NewTerms,Spec)->
+    EvalTerms = lists:map(fun(Mfa)->
+	{eval, Mfa}
+    end,
+    Mfas),
+    evaluate([EvalTerms|Ts], NewTerms, Spec);
+evaluate([{eval,Node,{M,F,Args}}|Ts],NewTerms,Spec) ->
     case rpc:call(Node,M,F,Args) of
 	{badrpc,Reason} ->
 	    throw({error,{Node,{M,F,length(Args)},Reason}});
 	_ ->
 	    ok
     end,
-    evaluate(Ts,Spec);
-evaluate([{eval,{M,F,Args}}|Ts],Spec) ->
+    evaluate(Ts,NewTerms,Spec);
+evaluate([{eval,{M,F,Args}}|Ts],NewTerms,Spec) ->
     case catch apply(M,F,Args) of
 	{'EXIT',Reason} ->
 	    throw({error,{node(),{M,F,length(Args)},Reason}});
 	_ ->
 	    ok
     end,
-    evaluate(Ts,Spec);
-evaluate([],_Spec) ->
-    ok.
+    evaluate(Ts,NewTerms,Spec);
+evaluate([Term|Ts], NewTerms, Spec)->
+    evaluate(Ts, [Term|NewTerms], Spec);
+evaluate([], NewTerms, _Spec) ->
+    NewTerms.
 
 get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) ->
     get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]});
@@ -373,6 +392,65 @@ get_all_nodes([_|Ts],Spec) ->
 get_all_nodes([],Spec) ->
     Spec.
 
+filter_nodestart_specs([{node_start, Options}|Ts], NewTerms, Spec) ->
+    filter_nodestart_specs([{node_start, list_nodes(Spec), Options}|Ts], NewTerms, Spec);
+filter_nodestart_specs([{node_start, NodeRef, Options}|Ts], NewTerms, Spec) when is_atom(NodeRef) ->
+    filter_nodestart_specs([{node_start, [NodeRef], Options}|Ts], NewTerms, Spec);
+filter_nodestart_specs([{node_start, NodeRefs, Options}|Ts], NewTerms, Spec=#testspec{node_start=NodeStart})->
+    Options2 = case lists:keyfind(callback_module, 1, Options) of
+	false->
+	    [{callback_module, ct_slave}|Options];
+	{callback_module, _Callback}->
+	    Options
+    end,
+    NSSAdder = fun(NodeRef, NodeStartAcc)->
+	Node=ref2node(NodeRef,Spec#testspec.nodes),
+        case lists:keyfind(Node, 1, NodeStartAcc) of
+	    false->
+		[{Node, Options2}|NodeStartAcc];
+	    {Node, OtherOptions}->
+		io:format("~nWarning: There are other options defined for node ~p:"
+			  "~n~w, skipping ~n~w...~n", [Node, OtherOptions, Options2]),
+		NodeStartAcc
+	end
+    end,
+    NodeStart2 = lists:foldl(NSSAdder, NodeStart, NodeRefs),
+    filter_nodestart_specs(Ts, NewTerms, Spec#testspec{node_start=NodeStart2});
+filter_nodestart_specs([Term|Ts], NewTerms, Spec)->
+    filter_nodestart_specs(Ts, [Term|NewTerms], Spec);
+filter_nodestart_specs([], NewTerms, Spec) ->
+    {NewTerms, Spec}.
+
+filter_evals([{eval,NodeRefs,Mfa}|Ts], NewTerms, Spec) when is_list(NodeRefs)->
+    EvalTerms = lists:map(fun(NodeRef)->
+			      {eval, NodeRef, Mfa}
+			  end,
+			  NodeRefs),
+    filter_evals(EvalTerms++Ts, NewTerms, Spec);
+filter_evals([{eval,NodeRef,{_,_,_}=Mfa}|Ts],NewTerms,Spec)->
+    filter_evals([{eval,NodeRef,[Mfa]}|Ts],NewTerms,Spec);
+filter_evals([{eval,NodeRef,[{_,_,_}|_]=Mfas}=EvalTerm|Ts],
+	     NewTerms,Spec=#testspec{node_start=NodeStart})->
+    Node=ref2node(NodeRef,Spec#testspec.nodes),
+    case lists:keyfind(Node, 1, NodeStart) of
+	false->
+	    filter_evals(Ts, [EvalTerm|NewTerms], Spec);
+	{Node, Options}->
+	    Options2 = case lists:keyfind(startup_functions, 1, Options) of
+		false->
+		     [{startup_functions, Mfas}|Options];
+		{startup_functions, StartupFunctions}->
+		     lists:keyreplace(startup_functions, 1, Options,
+			 {startup_functions, StartupFunctions ++ Mfas})
+	    end,
+	    NodeStart2 = lists:keyreplace(Node, 1, NodeStart, {Node, Options2}),
+	    filter_evals(Ts, NewTerms, Spec#testspec{node_start=NodeStart2})
+    end;
+filter_evals([Term|Ts], NewTerms, Spec)->
+    filter_evals(Ts, [Term|NewTerms], Spec);
+filter_evals([], NewTerms, Spec)->
+    {NewTerms, Spec}.
+
 save_nodes(Nodes,Spec=#testspec{nodes=NodeRefs}) ->
     NodeRefs1 =
 	lists:foldr(fun(all_nodes,NR) ->
@@ -794,6 +872,8 @@ valid_terms() ->
      {cover,3},
      {config,2},
      {config,3},
+     {userconfig, 2},
+     {userconfig, 3},
      {alias,3},
      {logdir,2},
      {logdir,3},
@@ -802,7 +882,6 @@ valid_terms() ->
      {event_handler,4},
      {include,2},
      {include,3},
-
      {suites,3},
      {suites,4},
      {cases,4},
diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl
index c2f51dfba5..7c8fd67e1d 100644
--- a/lib/common_test/src/ct_util.hrl
+++ b/lib/common_test/src/ct_util.hrl
@@ -29,6 +29,7 @@
 
 -record(testspec, {spec_dir,
 	           nodes=[],
+		   node_start=[],
 		   logdir=["."],
 		   cover=[],
 		   config=[],