diff options
| -rw-r--r-- | lib/common_test/doc/src/Makefile | 3 | ||||
| -rw-r--r-- | lib/common_test/doc/src/ref_man.xml | 1 | ||||
| -rw-r--r-- | lib/common_test/src/Makefile | 3 | ||||
| -rw-r--r-- | lib/common_test/src/common_test.app.src | 4 | ||||
| -rw-r--r-- | lib/common_test/src/ct_slave.erl | 416 | ||||
| -rw-r--r-- | lib/common_test/test/ct_master_SUITE.erl | 91 |
6 files changed, 444 insertions, 74 deletions
diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index a2c014418d..6322860088 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -45,7 +45,8 @@ CT_MODULES = \ ct_ssh \ ct_rpc \ ct_snmp \ - unix_telnet + unix_telnet \ + ct_slave CT_XML_FILES = $(CT_MODULES:=.xml) diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index beb3ed3247..5f84f75acb 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -75,6 +75,7 @@ <xi:include href="ct_snmp.xml"/> <xi:include href="ct_telnet.xml"/> <xi:include href="unix_telnet.xml"/> + <xi:include href="ct_slave.xml"/> </application> diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 4c4382b705..76646c8623 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -66,7 +66,8 @@ MODULES= \ unix_telnet \ ct_config \ ct_config_plain \ - ct_config_xml + ct_config_xml \ + ct_slave TARGET_MODULES= $(MODULES:%=$(EBIN)/%) diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 2f599a7e30..b00aafc87e 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -45,10 +45,12 @@ vts, ct_config, ct_config_plain, - ct_config_xml + ct_config_xml, + ct_slave ]}, {registered, [ct_logs, ct_util_server, + ct_config_server, ct_make_ref, vts, ct_master, diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl new file mode 100644 index 0000000000..57170220b8 --- /dev/null +++ b/lib/common_test/src/ct_slave.erl @@ -0,0 +1,416 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%%---------------------------------------------------------------------- +%% File : ct_slave.erl +%% Description : CT module for starting nodes for large-scale testing. +%% +%% Created : 7 April 2010 +%%---------------------------------------------------------------------- +-module(ct_slave). + +-export([start/1, start/2, start/3, stop/1, stop/2]). + +-export([slave_started/2, slave_ready/2, monitor_master/1]). + +-record(options, {username, password, boot_timeout, init_timeout, + startup_timeout, startup_functions, monitor_master, + kill_if_fail}). + +%%%----------------------------------------------------------------- +%%% @spec start(Node) -> Result +%%% Node = atom() +%%% Result = {ok, NodeName} | +%%% {error, already_started, NodeName} | +%%% {error, started_not_connected, NodeName} | +%%% {error, boot_timeout, NodeName} | +%%% {error, init_timeout, NodeName} | +%%% {error, startup_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Starts an Erlang node with name <code>Node</code> on the local host. +%%% @see start/3 +start(Node)-> + start(gethostname(), Node). + +%%%----------------------------------------------------------------- +%%% @spec start(Host, Node) -> Result +%%% Node = atom() +%%% Host = atom() +%%% Result = {ok, NodeName} | +%%% {error, already_started, NodeName} | +%%% {error, started_not_connected, NodeName} | +%%% {error, boot_timeout, NodeName} | +%%% {error, init_timeout, NodeName} | +%%% {error, startup_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Starts an Erlang node with name <code>Node</code> on host +%%% <code>Host</code> with the default options. +%%% @see start/3 +start(Host, Node)-> + start(Host, Node, []). + +%%%----------------------------------------------------------------- +%%% @spec start(Host, Node, Opts) -> Result +%%% Node = atom() +%%% Host = atom() +%%% Opts = [OptTuples] +%%% OptTuples = {username, Username} | +%%% {password, Password} | +%%% {boot_timeout, BootTimeout} | {init_timeout, InitTimeout} | +%%% {startup_timeout, StartupTimeout} | +%%% {startup_functions, StartupFunctions} | +%%% {monitor_master, Monitor} | +%%% {kill_if_fail, KillIfFail} +%%% Username = string() +%%% Password = string() +%%% BootTimeout = integer() +%%% InitTimeout = integer() +%%% StartupTimeout = integer() +%%% StartupFunctions = [StartupFunctionSpec] +%%% StartupFunctionSpec = {Module, Function, Arguments} +%%% Module = atom() +%%% Function = atom() +%%% Arguments = [term] +%%% Monitor = bool +%%% KillIfFail = bool +%%% Result = {ok, NodeName} | {error, already_started, NodeName} | +%%% {error, started_not_connected, NodeName} | +%%% {error, boot_timeout, NodeName} | +%%% {error, init_timeout, NodeName} | +%%% {error, startup_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Starts an Erlang node with name <code>Node</code> on host +%%% <code>Host</code> as specified by the combination of options in +%%% <code>Opts</code>. +%%% +%%% <p>Options <code>Username</code> and <code>Password</code> will be used +%%% to log in onto the remote host <code>Host</code>. +%%% Username, if omitted, defaults to the current user name, +%%% and password is empty by default.</p> +%%% +%%% <p>A list of functions specified in the <code>Startup</code> option will be +%%% executed after startup of the node. Note that all used modules should be +%%% present in the code path on the <code>Host</code>.</p> +%%% +%%% <p>The timeouts are applied as follows: +%%% <list> +%%% <item> +%%% <code>BootTimeout</code> - time to start the Erlang node, in seconds. +%%% Defaults to 3 seconds. If node does not become pingable within this time, +%%% the result <code>{error, boot_timeout, NodeName}</code> is returned; +%%% </item> +%%% <item> +%%% <code>InitTimeout</code> - time to wait for the node until it calls the +%%% internal callback function informing master about successfull startup. +%%% Defaults to one second. +%%% In case of timed out message the result +%%% <code>{error, init_timeout, NodeName}</code> is returned; +%%% </item> +%%% <item> +%%% <code>StartupTimeout</code> - time to wait intil the node finishes to run +%%% the <code>StartupFunctions</code>. Defaults to one second. +%%% If this timeout occurs, the result +%%% <code>{error, startup_timeout, NodeName}</code> is returned. +%%% </item> +%%% </list></p> +%%% +%%% <p>Option <code>monitor_master</code> specifies, if the slave node should be +%%% stopped in case of master node stop. Defaults to false</p> +%%% +%%% <p>Option <code>kill_if_fail</code> specifies, if the slave node should be +%%% killed in case of a timeout during initialization or startup. +%%% Defaults to true. Note that node also may be still alive it the boot +%%% timeout occurred, but it will not be killed in this case.</p> +%%% +%%% <p>Special return values are: +%%% <list> +%%% <item><code>{error, already_started, NodeName}</code> - if the node with +%%% the given name is already started on a given host;</item> +%%% <item><code>{error, started_not_connected, NodeName}</code> - if node is +%%% started, but not connected to the master node.</item> +%%% </list></p> +%%% +start(Host, Node, Options)-> + ENode = enodename(Host, Node), + case net_kernel:longnames() of + ignored-> + {error, not_alive}; + _-> + case is_started(ENode) of + false-> + OptionsRec = fetch_options(Options), + do_start(Host, Node, OptionsRec); + {true, not_connected}-> + {error, started_not_connected, ENode}; + {true, connected}-> + {error, already_started, ENode} + end + end. + +%%% @spec stop(Node) -> Result +%%% Node = atom() +%%% Result = {ok, NodeName} | +%%% {error, not_started, NodeName} | +%%% {error, not_connected, NodeName} | +%%% {error, stop_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Stops the running Erlang node with name <code>Node</code> on +%%% the localhost. +stop(Node)-> + stop(gethostname(), Node). + +%%% @spec stop(Host, Node) -> Result +%%% Host = atom() +%%% Node = atom() +%%% Result = {ok, NodeName} | +%%% {error, not_started, NodeName} | +%%% {error, not_connected, NodeName} | +%%% {error, stop_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Stops the running Erlang node with name <code>Node</code> on +%%% host <code>Host</code>. +stop(Host, Node)-> + ENode = enodename(Host, Node), + case is_started(ENode) of + {true, connected}-> + do_stop(ENode); + {true, not_connected}-> + {error, not_connected, ENode}; + false-> + {error, not_started, ENode} + end. + +%%% fetch an option value from the tagged tuple list with default +get_option_value(Key, OptionList, Default)-> + case lists:keyfind(Key, 1, OptionList) of + false-> + Default; + {Key, Value}-> + Value + end. + +%%% convert option list to the option record, fill all defaults +fetch_options(Options)-> + UserName = get_option_value(username, Options, []), + Password = get_option_value(password, Options, []), + BootTimeout = get_option_value(boot_timeout, Options, 3), + InitTimeout = get_option_value(init_timeout, Options, 1), + StartupTimeout = get_option_value(startup_timeout, Options, 1), + StartupFunctions = get_option_value(startup_functions, Options, []), + Monitor = get_option_value(monitor_master, Options, false), + KillIfFail = get_option_value(kill_if_fail, Options, true), + #options{username=UserName, password=Password, + boot_timeout=BootTimeout, init_timeout=InitTimeout, + startup_timeout=StartupTimeout, startup_functions=StartupFunctions, + monitor_master=Monitor, kill_if_fail=KillIfFail}. + +% send a message when slave node is started +% @hidden +slave_started(ENode, MasterPid)-> + MasterPid ! {node_started, ENode}, + ok. + +% send a message when slave node has finished startup +% @hidden +slave_ready(ENode, MasterPid)-> + MasterPid ! {node_ready, ENode}, + ok. + +% start monitoring of the master node +% @hidden +monitor_master(MasterNode)-> + spawn(fun()->monitor_master_int(MasterNode) end). + +% code of the masterdeath-waiter process +monitor_master_int(MasterNode)-> + erlang:monitor_node(MasterNode, true), + receive + {nodedown, MasterNode}-> + init:stop() + end. + +% check if node is listed in the nodes() +is_connected(ENode)-> + [N||N<-nodes(), N==ENode] == [ENode]. + +% check if node is alive (ping and disconnect if pingable) +is_started(ENode)-> + case is_connected(ENode) of + true-> + {true, connected}; + false-> + case net_adm:ping(ENode) of + pang-> + false; + pong-> + erlang:disconnect_node(ENode), + {true, not_connected} + end + end. + +% make a Erlang node name from name and hostname +enodename(Host, Node)-> + list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)). + +% performs actual start of the "slave" node +do_start(Host, Node, Options)-> + ENode = enodename(Host, Node), + Functions = + lists:concat([[{ct_slave, slave_started, [ENode, self()]}], + Options#options.startup_functions, + [{ct_slave, slave_ready, [ENode, self()]}]]), + Functions2 = if + Options#options.monitor_master-> + [{ct_slave, monitor_master, [node()]}|Functions]; + true-> + Functions + end, + MasterHost = gethostname(), + if + MasterHost == Host -> + spawn_local_node(Node); + true-> + spawn_remote_node(Host, Node, Options) + end, + BootTimeout = Options#options.boot_timeout, + InitTimeout = Options#options.init_timeout, + StartupTimeout = Options#options.startup_timeout, + Result = case wait_for_node_alive(ENode, BootTimeout) of + pong-> + call_functions(ENode, Functions2), + receive + {node_started, ENode}-> + receive + {node_ready, ENode}-> + {ok, ENode} + after StartupTimeout*1000-> + {error, startup_timeout, ENode} + end + after InitTimeout*1000 -> + {error, init_timeout, ENode} + end; + pang-> + {error, boot_timeout, ENode} + end, + case Result of + {ok, ENode}-> + ok; + {error, Timeout, ENode} + when ((Timeout==init_timeout) or (Timeout==startup_timeout)) and + Options#options.kill_if_fail-> + do_stop(ENode); + _-> ok + end, + Result. + +% are we using fully qualified hostnames +long_or_short()-> + case net_kernel:longnames() of + true-> + "-name"; + false-> + "-sname" + end. + +% get the localhost's name, depending on the using name policy +gethostname()-> + Hostname = case net_kernel:longnames() of + true-> + net_adm:localhost(); + _-> + {ok, Name}=inet:gethostname(), + Name + end, + list_to_atom(Hostname). + +% get cmd for starting Erlang +get_cmd(Node)-> + "erl -detached -noinput "++ long_or_short() ++ " " ++ atom_to_list(Node). + +% spawn node locally +spawn_local_node(Node)-> + Cmd = get_cmd(Node), + %io:format("Running cmd: ~p~n", [Cmd]), + open_port({spawn, Cmd}, [stream]). + +% start crypto and ssh if not yet started +check_for_ssh_running()-> + case application:get_application(crypto) of + undefined-> + application:start(crypto), + case application:get_application(ssh) of + undefined-> + application:start(ssh); + {ok, ssh}-> + ok + end; + {ok, crypto}-> + ok + end. + +% spawn node remotely +spawn_remote_node(Host, Node, Options)-> + Username = Options#options.username, + Password = Options#options.password, + SSHOptions = case {Username, Password} of + {[], []}-> + []; + {_, []}-> + [{user, Username}]; + {_, _}-> + [{user, Username}, {password, Password}] + end ++ [{silently_accept_hosts, true}], + check_for_ssh_running(), + {ok, SSHConnRef} = ssh:connect(atom_to_list(Host), 22, SSHOptions), + {ok, SSHChannelId} = ssh_connection:session_channel(SSHConnRef, infinity), + ssh_connection:exec(SSHConnRef, SSHChannelId, get_cmd(Node), infinity). + +% call functions on a remote Erlang node +call_functions(_Node, [])-> + ok; +call_functions(Node, [{M, F, A}|Functions])-> + rpc:call(Node, M, F, A), + call_functions(Node, Functions). + +% wait N seconds until node is pingable +wait_for_node_alive(_Node, 0)-> + pang; +wait_for_node_alive(Node, N)-> + timer:sleep(1000), + case net_adm:ping(Node) of + pong-> + pong; + pang-> + wait_for_node_alive(Node, N-1) + end. + +% call init:stop on a remote node +do_stop(ENode)-> + spawn(ENode, init, stop, []), + wait_for_node_dead(ENode, 5). + +% wait N seconds until node is disconnected +wait_for_node_dead(Node, 0)-> + {error, stop_timeout, Node}; +wait_for_node_dead(Node, N)-> + timer:sleep(1000), + case lists:member(Node, nodes()) of + true-> + wait_for_node_dead(Node, N-1); + false-> + {ok, Node} + end. diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl index a2eaf98e34..ebd399ad64 100644 --- a/lib/common_test/test/ct_master_SUITE.erl +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -60,37 +60,23 @@ all(doc) -> all(suite) -> [ - ct_master_test_peer, - ct_master_test_slave + ct_master_test ]. %%-------------------------------------------------------------------- %% TEST CASES %%-------------------------------------------------------------------- -ct_master_test_peer(Config) when is_list(Config)-> +ct_master_test(Config) when is_list(Config)-> NodeCount = 5, DataDir = ?config(data_dir, Config), NodeNames = [list_to_atom("testnode_"++integer_to_list(N)) || N <- lists:seq(1, NodeCount)], FileName = filename:join(DataDir, "ct_master_spec.spec"), Suites = [master_SUITE], - make_spec(DataDir, FileName, NodeNames, Suites, Config), - start_nodes(NodeNames, peer), - run_test(ct_master_test, FileName, Config), - stop_nodes(NodeNames, peer), - file:delete(filename:join(DataDir, FileName)). - -ct_master_test_slave(Config) when is_list(Config)-> - NodeCount = 5, - DataDir = ?config(data_dir, Config), - NodeNames = [list_to_atom("testnode_"++integer_to_list(N)) || - N <- lists:seq(1, NodeCount)], - FileName = filename:join(DataDir, "ct_master_spec.spec"), - Suites = [master_SUITE], - make_spec(DataDir, FileName, NodeNames, Suites, Config), - start_nodes(NodeNames, slave), - run_test(ct_master_test, FileName, Config), - stop_nodes(NodeNames, slave), + TSFile = make_spec(DataDir, FileName, NodeNames, Suites, Config), + start_nodes(NodeNames), + [{TSFile, ok}] = run_test(ct_master_test, FileName, Config), + stop_nodes(NodeNames), file:delete(filename:join(DataDir, FileName)). %%%----------------------------------------------------------------- @@ -100,7 +86,7 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config)-> {ok, HostName} = inet:gethostname(), N = lists:map(fun(NodeName)-> - {node, NodeName, enodename(HostName, NodeName)} + {node, NodeName, list_to_atom(atom_to_list(NodeName)++"@"++HostName)} end, NodeNames), @@ -117,67 +103,30 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config)-> S = [{suites, NodeNames, filename:join(DataDir, "master"), Suites}], PrivDir = ?config(priv_dir, Config), - LD = [{logdir, PrivDir}, {logdir, master, PrivDir}], + LD = lists:map(fun(NodeName)-> + {logdir, NodeName, get_log_dir(PrivDir, NodeName)} + end, + NodeNames) ++ [{logdir, master, PrivDir}], ct_test_support:write_testspec(N++C++S++LD, DataDir, FileName). +get_log_dir(PrivDir, NodeName)-> + LogDir = filename:join(PrivDir, io_lib:format("slave.~p", [NodeName])), + file:make_dir(LogDir), + LogDir. + run_test(_Name, FileName, Config)-> [{FileName, ok}] = ct_test_support:run(ct_master, run, [FileName], Config). -wait_for_node_alive(_Node, 0)-> - pang; -wait_for_node_alive(Node, N)-> - timer:sleep(1000), - case net_adm:ping(Node) of - pong-> - pong; - pang-> - wait_for_node_alive(Node, N-1) - end. - -wait_for_node_dead(_Node, 0)-> - error; -wait_for_node_dead(Node, N)-> - timer:sleep(1000), - case lists:member(Node, nodes()) of - true-> - wait_for_node_dead(Node, N-1); - false-> - ok - end. - -enodename(HostName, NodeName)-> - list_to_atom(atom_to_list(NodeName)++"@"++HostName). - -start_node(HostName, NodeName, peer)-> - ENodeName = enodename(HostName, NodeName), - Cmd = "erl -detached -noinput -sname "++atom_to_list(NodeName), - open_port({spawn, Cmd}, [stream]), - pong = wait_for_node_alive(ENodeName, 3); -start_node(HostName, NodeName, slave)-> - ENodeName = enodename(HostName, NodeName), - {ok, ENodeName} = - slave:start(list_to_atom(HostName), NodeName). - -stop_node(HostName, NodeName, peer)-> - ENodeName = enodename(HostName, NodeName), - spawn(ENodeName, init, stop, []), - wait_for_node_dead(ENodeName, 3); -stop_node(HostName, NodeName, slave)-> - ENodeName = enodename(HostName, NodeName), - ok = slave:stop(ENodeName). - -start_nodes(NodeNames, Type)-> - {ok, HostName} = inet:gethostname(), +start_nodes(NodeNames)-> lists:foreach(fun(NodeName)-> - start_node(HostName, NodeName, Type) + {ok, _}=ct_slave:start(NodeName) end, NodeNames). -stop_nodes(NodeNames, Type)-> - {ok, HostName} = inet:gethostname(), +stop_nodes(NodeNames)-> lists:foreach(fun(NodeName)-> - stop_node(HostName, NodeName, Type) + {ok, _}=ct_slave:stop(NodeName) end, NodeNames). |