diff options
Diffstat (limited to 'lib/common_test')
86 files changed, 10019 insertions, 2037 deletions
diff --git a/lib/common_test/Makefile b/lib/common_test/Makefile index ebca4523ab..aecec1a50d 100644 --- a/lib/common_test/Makefile +++ b/lib/common_test/Makefile @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index a2c014418d..6322860088 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -45,7 +45,8 @@ CT_MODULES = \ ct_ssh \ ct_rpc \ ct_snmp \ - unix_telnet + unix_telnet \ + ct_slave CT_XML_FILES = $(CT_MODULES:=.xml) diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 7b52883f8a..e30eef2488 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Common Test</title> @@ -337,7 +337,7 @@ </func> <func> - <name>Module:testcase() -> [Info] </name> + <name>Module:Testcase() -> [Info] </name> <fsummary>Test case info function. </fsummary> <type> <v> Info = {timetrap,Time} | {require,Required} | @@ -396,7 +396,7 @@ <func> - <name>Module:testcase(Config) -> void() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() </name> + <name>Module:Testcase(Config) -> void() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() </name> <fsummary>A test case</fsummary> <type> <v> Config = SaveConfig = [{Key,Value}]</v> diff --git a/lib/common_test/doc/src/config_file_chapter.xml b/lib/common_test/doc/src/config_file_chapter.xml index a22a5270c1..850bc74e19 100644 --- a/lib/common_test/doc/src/config_file_chapter.xml +++ b/lib/common_test/doc/src/config_file_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2009</year> + <year>2004</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Config Files</title> @@ -180,7 +180,126 @@ </section> <section> - <title>Examples</title> + <title>User specific configuration data formats</title> + + <p>It is possible for the user to specify configuration data on a + different format than key-value tuples in a text file, as described + so far. The data can e.g. be read from arbitrary files, fetched from + the web over http, or requested from a user specific process. + To support this, Common Test provides a callback module plugin + mechanism to handle configuration data.</p> + + <section> + <title>Default callback modules for handling configuration data</title> + <p>The Common Test application includes default callback modules + for handling configuration data specified in standard config files + (see above) and in xml files:</p> + <list> + <item> + <c>ct_config_plain</c> - for reading configuration files with + key-value tuples (standard format). This handler will be used to + parse configuration files if no user callback is specified. + </item> + <item> + <c>ct_config_xml</c> - for reading configuration data from XML + files. + </item> + </list> + </section> + + <section> + <title>Using XML configuration files</title> + <p>This is an example of an XML configuration file:</p> + <pre><![CDATA[ +<config> + <ftp_host> + <ftp>"targethost"</ftp> + <username>"tester"</username> + <password>"letmein"</password> + </ftp_host> + <lm_directory>"/test/loadmodules"</lm_directory> +</config>]]></pre> + + <p>This configuration file, once read, will produce the same configuration + variables as the following text file:</p> + <pre> +{ftp_host, [{ftp,"targethost"}, + {username,"tester"}, + {password,"letmein"}]}. + +{lm_directory, "/test/loadmodules"}.</pre> + </section> + + <section> + <title>How to implement a user specific handler</title> + + <p>The user specific handler can be written to handle special + configuration file formats. The parameter can be either file + name(s) or configuration string(s) (the empty list is valid).</p> + + <p>The callback module implementing the handler is responsible for + checking correctness of configuration strings.</p> + + <p>To perform validation of the configuration strings, the callback module + should have the following function exported:</p> + + <p><c>Callback:check_parameter/1</c></p> + <p>The input argument will be passed from Common Test, as defined in the test + specification or given as an option to <c>run_test</c>.</p> + + <p>The return value should be any of the following values indicating if given + configuration parameter is valid:</p> + <list> + <item> + <c>{ok, {file, FileName}}</c> - parameter is a file name and + the file exists, + </item> + <item> + <c>{ok, {config, ConfigString}}</c> - parameter is a config string + and it is correct, + </item> + <item> + <c>{error, {nofile, FileName}}</c> - there is no file with the given + name in the current directory, + </item> + <item> + <c>{error, {wrong_config, ConfigString}}</c> - the configuration string + is wrong. + </item> + </list> + + <p>To perform reading of configuration data - initially before the tests + start, or as a result of data being reloaded during test execution - + the following function should be exported from the callback module:</p> + + <p><c>Callback:read_config/1</c></p> + + <p>The input argument is the same as for the <c>check_parameter/1</c> function.</p> + <p>The return value should be either:</p> + + <list> + <item> + <c>{ok, Config}</c> - if the configuration variables are read successfully, + </item> + <item> + <c>{error, Error, ErrorDetails}</c> - if the callback module fails to + proceed with the given configuration parameters. + </item> + </list> + <p><c>Config</c> is the proper Erlang key-value list, with possible + key-value sublists as values, like for the configuration file + example above:</p> + + <pre> + [{ftp_host, [{ftp, "targethost"}, {username, "tester"}, {password, "letmein"}]}, + {lm_directory, "/test/loadmodules"}]</pre> + + </section> + + </section> + + <section> + <title>Examples of configuration data handling</title> <p>A config file for using the FTP client to access files on a remote host could look like this:</p> @@ -188,28 +307,33 @@ <pre> {ftp_host, [{ftp,"targethost"}, {username,"tester"}, - {password,"letmein"}]}. + {password,"letmein"}]}. {lm_directory, "/test/loadmodules"}.</pre> - <p>Example of how to assert that the configuration data is available and + + <p>The XML version shown in the chapter above can also be used, but it should be + explicitly specified that the <c>ct_config_xml</c> callback module is to be + used by Common Test.</p> + + <p>Example of how to assert that the configuration data is available and use it for an FTP session:</p> <pre> init_per_testcase(ftptest, Config) -> {ok,_} = ct_ftp:open(ftp), - Config. + Config. end_per_testcase(ftptest, _Config) -> ct_ftp:close(ftp). ftptest() -> [{require,ftp,ftp_host}, - {require,lm_directory}]. + {require,lm_directory}]. ftptest(Config) -> - Remote = filename:join(ct:get_config(lm_directory), "loadmodX"), + Remote = filename:join(ct:get_config(lm_directory), "loadmodX"), Local = filename:join(?config(priv_dir,Config), "loadmodule"), ok = ct_ftp:recv(ftp, Remote, Local), - ...</pre> + ...</pre> <p>An example of how the above functions could be rewritten if necessary to open multiple connections to the FTP server:</p> @@ -217,7 +341,7 @@ init_per_testcase(ftptest, Config) -> {ok,Handle1} = ct_ftp:open(ftp_host), {ok,Handle2} = ct_ftp:open(ftp_host), - [{ftp_handles,[Handle1,Handle2]} | Config]. + [{ftp_handles,[Handle1,Handle2]} | Config]. end_per_testcase(ftptest, Config) -> lists:foreach(fun(Handle) -> ct_ftp:close(Handle) end, @@ -225,17 +349,117 @@ ftptest() -> [{require,ftp_host}, - {require,lm_directory}]. + {require,lm_directory}]. ftptest(Config) -> - Remote = filename:join(ct:get_config(lm_directory), "loadmodX"), + Remote = filename:join(ct:get_config(lm_directory), "loadmodX"), Local = filename:join(?config(priv_dir,Config), "loadmodule"), [Handle | MoreHandles] = ?config(ftp_handles,Config), ok = ct_ftp:recv(Handle, Remote, Local), - ...</pre> + ...</pre> </section> + <section> + <title>Example of user specific configuration handler</title> + <p>A simple configuration handling driver which will ask an external server for + configuration data can be implemented this way:</p> + <pre> +-module(config_driver). +-export([read_config/1, check_parameter/1]). + +read_config(ServerName)-> + ServerModule = list_to_atom(ServerName), + ServerModule:start(), + ServerModule:get_config(). + +check_parameter(ServerName)-> + ServerModule = list_to_atom(ServerName), + case code:is_loaded(ServerModule) of + {file, _}-> + {ok, {config, ServerName}}; + false-> + case code:load_file(ServerModule) of + {module, ServerModule}-> + {ok, {config, ServerName}}; + {error, nofile}-> + {error, {wrong_config, "File not found: " ++ ServerName ++ ".beam"}} + end + end.</pre> + + <p>The configuration string for this driver may be "config_server", if the + config_server.erl module below is compiled and exists in the code path + during test execution:</p> + <pre> +-module(config_server). +-export([start/0, stop/0, init/1, get_config/0, loop/0]). + +-define(REGISTERED_NAME, ct_test_config_server). + +start()-> + case whereis(?REGISTERED_NAME) of + undefined-> + spawn(?MODULE, init, [?REGISTERED_NAME]), + wait(); + _Pid-> + ok + end, + ?REGISTERED_NAME. + +init(Name)-> + register(Name, self()), + loop(). + +get_config()-> + call(self(), get_config). + +stop()-> + call(self(), stop). + +call(Client, Request)-> + case whereis(?REGISTERED_NAME) of + undefined-> + {error, not_started, Request}; + Pid-> + Pid ! {Client, Request}, + receive + Reply-> + {ok, Reply} + after 4000-> + {error, timeout, Request} + end + end. + +loop()-> + receive + {Pid, stop}-> + Pid ! ok; + {Pid, get_config}-> + {D,T} = erlang:localtime(), + Pid ! + [{localtime, [{date, D}, {time, T}]}, + {node, erlang:node()}, + {now, erlang:now()}, + {config_server_pid, self()}, + {config_server_vsn, ?vsn}], + ?MODULE:loop() + end. + +wait()-> + case whereis(?REGISTERED_NAME) of + undefined-> + wait(); + _Pid-> + ok + end.</pre> + + <p>In this example, the handler also provides the ability to dynamically reload + configuration variables. If <c>ct:reload_config(localtime)</c> is called from + the test case function, all variables loaded with <c>config_driver:read_config/1</c> + will be updated with their latest values, and the new value for variable + <c>localtime</c> will be returned.</p> + </section> + </chapter> diff --git a/lib/common_test/doc/src/ct_master_chapter.xml b/lib/common_test/doc/src/ct_master_chapter.xml index 79288cfe4c..01f8e61d36 100644 --- a/lib/common_test/doc/src/ct_master_chapter.xml +++ b/lib/common_test/doc/src/ct_master_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2009</year> + <year>2006</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Using Common Test for Large Scale Testing</title> @@ -30,6 +30,7 @@ </header> <section> + <marker id="general"></marker> <title>General</title> <p>Large scale automated testing requires running multiple independent test sessions in parallel. This is accomplished by running @@ -102,9 +103,10 @@ <p><c>ct_master:abort()</c> (stop all) or <c>ct_master:abort(Nodes)</c></p> <p>For detailed information about the <c>ct_master</c> API, please see the - manual page for this module.</p> + <seealso marker="ct_master">manual page</seealso> for this module.</p> </section> <section> + <marker id="test_specifications"></marker> <title>Test Specifications</title> <p>The test specifications used as input to CT Master are fully compatible with the specifications used as input to the regular CT server. The syntax is described in the @@ -186,7 +188,7 @@ <seealso marker="run_test_chapter#test_specifications">Running Test Suites</seealso> chapter). The result is that any test specified to run on a node with the same name as the Common Test node in question (typically <c>ct@somehost</c> if started - with the <c>run_test</c> script), will be performed. Tests without explicit + with the <c>run_test</c> program), will be performed. Tests without explicit node association will always be performed too of course!</p> <note><p>It is recommended that absolute paths are used for log directories, @@ -194,6 +196,56 @@ current working directory settings are not important.</p></note> </section> + <section> + <title>Automatic startup of test target nodes</title> + <marker id="ct_slave"></marker> + <p>Is is possible to automatically start, and perform initial actions, on + test target nodes by using the test specification term <c>init</c>.</p> + <p>Currently, two sub-terms are supported, <c>node_start</c> and <c>eval</c>.</p> + <p>Example:</p> + <pre> + {node, node1, node1@host1}. + {node, node2, node1@host2}. + {node, node3, node2@host2}. + {node, node4, node1@host3}. + {init, node1, [{node_start, [{callback_module, my_slave_callback}]}]}. + {init, [node2, node3], {node_start, [{username, "ct_user"}, {password, "ct_password"}]}}. + {init, node4, {eval, {module, function, []}}}.</pre> + + <p>This test specification declares that <c>node1@host1</c> is to be started using + the user callback function <c>callback_module:my_slave_callback/0</c>, and nodes + <c>node1@host2</c> and <c>node2@host2</c> will be started with the default callback + module <c>ct_slave</c>. The given user name and password is used to log into remote + host <c>host2</c>. Also, the function <c>module:function/0</c> will be evaluated on + <c>node1@host3</c>, and the result of this call will be printed to the log.</p> + + <p>The default <seealso marker="ct_slave">ct_slave</seealso> callback module, + which is part of the Common Test application, has the following features: + <list> + <item>Starting Erlang target nodes on local or remote hosts + (ssh is used for communication). + </item> + <item>Ability to start an Erlang emulator with additional flags + (any flags supported by <c>erl</c> are supported). + </item> + <item>Supervision of a node being started by means of internal callback + functions. Used to prevent hanging nodes. (Configurable). + </item> + <item>Monitoring of the master node by the slaves. A slave node may be + stopped in case the master node terminates. (Configurable). + </item> + <item>Execution of user functions after a slave node is started. + Functions can be given as a list of {Module, Function, Arguments} tuples. + </item> + </list> + </p> + <p>Note that it is possible to specify an <c>eval</c> term for the node as well + as <c>startup_functions</c> in the <c>node_start</c> options list. In this + case first the node will be started, then the <c>startup_functions</c> are + executed, and finally functions specified with <c>eval</c> are called. + </p> + </section> + </chapter> diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index a550810850..7f5144b760 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2009</year> + <year>2006</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Event Handling</title> @@ -68,29 +68,27 @@ Example:</p> <p><c>$ run_test -suite test/my_SUITE -event_handler handlers/my_evh1 handlers/my_evh2 -pa $PWD/handlers</c></p> + <p>Use the <c><![CDATA[run_test -event_handler_init]]></c> option instead of + <c><![CDATA[-event_handler]]></c> to pass start arguments to the event handler + init function.</p> <p>All event handler modules must have gen_event behaviour. Note also that these modules must be precompiled, and that their locations must be added explicitly to the Erlang code server search path (like in the example).</p> - <p>It is not possible to specify start arguments to the event handlers when - using the <c>run_test</c> script. You may however pass along start arguments - if you use the <c>ct:run_test/1</c> function. An event_handler tuple in the argument - <c>Opts</c> has the following definition (see also <c>ct:run_test/1</c> in the - reference manual):</p> + <p>An event_handler tuple in the argument <c>Opts</c> has the following + definition (see also <c>ct:run_test/1</c> in the reference manual):</p> <pre> {event_handler,EventHandlers} EventHandlers = EH | [EH] EH = atom() | {atom(),InitArgs} | {[atom()],InitArgs} - InitArgs = [term()] - </pre> + InitArgs = [term()]</pre> <p>Example:</p> <pre> - 1> ct:run_test([{suite,"test/my_SUITE"},{event_handler,[my_evh1,{my_evh2,[node()]}]}]). - </pre> + 1> ct:run_test([{suite,"test/my_SUITE"},{event_handler,[my_evh1,{my_evh2,[node()]}]}]).</pre> <p>This will install two event handlers for the <c>my_SUITE</c> test. Event handler <c>my_evh1</c> is started with <c>[]</c> as argument to the init function. Event handler <c>my_evh2</c> is started with the name of the current node in the init argument list.</p> diff --git a/lib/common_test/doc/src/install_chapter.xml b/lib/common_test/doc/src/install_chapter.xml index e1ff5abf6a..828588a673 100644 --- a/lib/common_test/doc/src/install_chapter.xml +++ b/lib/common_test/doc/src/install_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2007</year><year>2009</year> + <year>2007</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Installation</title> @@ -33,82 +33,77 @@ <marker id="general"></marker> <title>General information</title> - <p>The two main interfaces for running tests with Common Test - are an executable Bourne shell script named <c>run_test</c> and an - erlang module named <c>ct</c>. The shell script will work on Unix/Linux - (and Linux-like environments such as Cygwin on Windows) and the - <c>ct</c> interface functions can be called from the Erlang shell - (or from any Erlang function) on any supported platform.</p> - - <p>The Common Test application is installed with the Erlang/OTP - system and no explicit installation is required to start using - Common Test by means of the interface functions in the <c>ct</c> - module. If you wish to use <c>run_test</c>, however, this script - needs to be generated first, according to the instructions below.</p> - </section> + <p>The two main interfaces for running tests with Common Test + are an executable program named run_test and an + erlang module named <c>ct</c>. The run_test program + is compiled for the underlying operating system (e.g. Unix/Linux + or Windows) during the build of the Erlang/OTP system, and is + installed automatically with other executable programs in + the top level <c>bin</c> directory of Erlang/OTP. + The <c>ct</c> interface functions can be called from the Erlang shell, + or from any Erlang function, on any supported platform.</p> + + <p>A legacy Bourne shell script - also named run_test - exists, + which may be manually generated and installed. This script may be used + instead of the run_test program mentioned above, e.g. if the user + wishes to modify or customize the Common Test start flags in a simpler + way than making changes to the run_test C program.</p> - <section> - <title>Unix/Linux</title> - - <p>Go to the <c><![CDATA[common_test-<vsn>]]></c> directory, located - among the other OTP applications (under the OTP lib directory). Here you - execute the <c>install.sh</c> script with argument <c>local</c>:</p> - - <p><c> - $ ./install.sh local - </c></p> + <p>The Common Test application is installed with the Erlang/OTP + system and no additional installation step is required to start using + Common Test by means of the run_test executable program, and/or the interface + functions in the <c>ct</c> module. If you wish to use the legacy Bourne + shell script version of run_test, however, this script needs to be + generated first, according to the instructions below.</p> + + <p><note>Before reading on, please note that since Common Test version + 1.5, the run_test shell script is no longer required for starting + tests with Common Test from the OS command line. The run_test + program (descibed above) is the new recommended command line interface + for Common Test. The shell script exists mainly for legacy reasons and + may not be updated in future releases of Common Test. It may even be removed. + </note></p> + + <p>Optional step to generate a shell script for starting Common Test:</p> + <p>To generate the run_test shell script, navigate to the + <c><![CDATA[common_test-<vsn>]]></c> directory, located among the other + OTP applications (under the OTP lib directory). Here execute the + <c>install.sh</c> script with argument <c>local</c>:</p> + + <p><c> + $ ./install.sh local + </c></p> - <p>This generates the executable <c>run_test</c> script in the - <c><![CDATA[common_test-<vsn>/priv/bin]]></c> directory. The script - will include absolute paths to the Common Test and Test Server - application directories, so it's possible to copy or move the script to - a different location on the file system, if desired, without having to - update it. It's of course possible to leave the script under the - <c>priv/bin</c> directory and update the PATH variable accordingly (or - create a link or alias to it).</p> - - <p>If you, for any reason, have copied Common Test and Test Server - to a different location than the default OTP lib directory, you can - generate a <c>run_test</c> script with a different top level directory, - simply by specifying the directory, instead of <c>local</c>, when running - <c>install.sh</c>. Example:</p> - - <p><c> - $ install.sh /usr/local/test_tools + <p>This generates the executable run_test script in the + <c><![CDATA[common_test-<vsn>/priv/bin]]></c> directory. The script + will include absolute paths to the Common Test and Test Server + application directories, so it's possible to copy or move the script to + a different location on the file system, if desired, without having to + update it. It's of course possible to leave the script under the + <c>priv/bin</c> directory and update the PATH variable accordingly (or + create a link or alias to it).</p> + + <p>If you, for any reason, have copied Common Test and Test Server + to a different location than the default OTP lib directory, you can + generate a run_test script with a different top level directory, + simply by specifying the directory, instead of <c>local</c>, when running + <c>install.sh</c>. Example:</p> + + <p><c> + $ install.sh /usr/local/test_tools </c></p> <p>Note that the <c><![CDATA[common_test-<vsn>]]></c> and - <c><![CDATA[test_server-<vsn>]]></c> directories must be located under the - same top directory. Note also that the install script does not copy files - or update environment variables. It only generates the <c>run_test</c> - script.</p> + <c><![CDATA[test_server-<vsn>]]></c> directories must be located under the + same top directory. Note also that the install script does not copy files + or update environment variables. It only generates the run_test + script.</p> - <p>Whenever you install a new version of Erlang/OTP, the <c>run_test</c> - script needs to be regenerated, or updated manually with new directory names - (new version numbers), for it to "see" the latest Common Test and Test Server - versions.</p> + <p>Whenever you install a new version of Erlang/OTP, the run_test + script needs to be regenerated, or updated manually with new directory names + (new version numbers), for it to "see" the latest Common Test and Test Server + versions.</p> - <p>For more information on the <c>run_test</c> script and the <c>ct</c> - module, please see the reference manual.</p> - </section> - - <section> - <title>Windows</title> - - <p>On Windows it is very convenient to use Cygwin (<c>www.cygwin.com</c>) - for running Common Test and Erlang, since it enables you to use the - <c>run_test</c> script for starting Common Test. If you are a Cygwin - user, simply follow the instructions above for generating the <c>run_test</c> - script.</p> - - <p>If you do not use Cygwin, you have to rely on the API functions - in the <c>ct</c> module (instead of <c>run_test</c>) for running - Common Test as described initially in this chapter.</p> - - <p>If you, for any reason, have chosen to store Common Test and Test Server - in a different location than the default OTP lib directory, make - sure the <c>ebin</c> directories of these applications are included - in the Erlang code server path (so the application modules can be loaded).</p> </section> </chapter> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 4f5f6caa8c..eadfc83c07 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -32,6 +32,152 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Process calls using monitors in Common Test would not + clear the inbox of remaining DOWN messages. This has been + fixed.</p> + <p> + Own Id: OTP-8621 Aux Id: seq11560 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + It is now possible for the user to provide specific + callback modules that handle test configuration data, so + that data on arbitray form can be accessed (e.g. by + reading files or by communicating with a configuration + server process). Two default callback modules have been + introduced in Common Test: ct_config_plain and + ct_config_xml. The former is used to handle the + traditional Common Test configuration files (with terms + on key-value tuple form) and the latter to handle + configuration data on XML representation.</p> + <p> + Own Id: OTP-8485</p> + </item> + <item> + <p> + It is now possible to execute test suites that are not + necessarily available on the local file system, but have + been loaded on the test node in advance (e.g. sent as + binaries from a remote node and loaded by RPC). A + requirement is that the no_auto_compile (or + {auto_compile,false}) parameter has been set.</p> + <p> + Own Id: OTP-8490 Aux Id: seq11500 </p> + </item> + <item> + <p> + Test Server will now call the end_per_testcase/2 function + even if the test case has been terminated explicitly + (with abort_current_testcase/1), or after a timetrap + timeout. Under these circumstances the return value of + end_per_testcase is completely ignored. Therefore the + function will not be able to change the reason for test + case termination by returning {fail,Reason}, nor will it + be able to save data with {save_config,Data}.</p> + <p> + Own Id: OTP-8500 Aux Id: seq11521 </p> + </item> + <item> + <p> + It is now possible to use the test specification term + 'init' to start Common Test nodes automatically, as well + as have initial function calls evaluated on the nodes. A + default callback module for the 'init' term, ct_slave, + has been introduced to enable Common Test Master to + perform host login and node startup operations over ssh.</p> + <p> + Own Id: OTP-8570</p> + </item> + <item> + <p> + The run_test script has been replaced by a program (with + the same name) which can be executed without explicit + installation. The start flags are the same as for the + legacy start script.</p> + <p> + Own Id: OTP-8650</p> + </item> + <item> + <p> + Previously, a repeat property of a test case group + specified the number of times the group should be + repeated after the main test run. I.e. {repeat,N} would + case the group to execute 1+N times. To be consistent + with the behaviour of the run_test repeat option, this + has been changed. N now specifies the absolute number of + executions instead.</p> + <p> + Own Id: OTP-8689 Aux Id: seq11502 </p> + </item> + <item> + <p> + With the run_test -erl_args option, it's possible to + divide the options on the run_test command line into ones + that Common Test should process (those preceding + -erl_args, and ones it should ignore (those succeeding + -erl_args). Options preceding -erl_args that Common Test + doesn't recognize are also ignored (i.e. the same + behaviour as earlier versions of Common Test).</p> + <p> + Own Id: OTP-8690 Aux Id: OTP-8650 </p> + </item> + <item> + <p> + Directories added with -pa or -pz in the pre-erl_args + part of the run_test command line will be converted from + relative to absolute, this to avoid problems loading user + modules when Common Test switches working directory + during the test run.</p> + <p> + Own Id: OTP-8691 Aux Id: OTP-8650 </p> + </item> + <item> + <p> + The timetrap handling has been made more user + controllable by means of new start options and new ct + interface functions. With the 'multiply_timetraps' start + option, it's possible to specify a value which all + timetrap timeout values get multiplied by. This is useful + e.g. to extend the timetraps temporarily while running + cover or trace. The 'scale_timetraps' start option + switches on or off the Test Server timetrap scaling + feature (which tries to detect if the tests may benefit + from extended timetraps, e.g. due to running certain test + tools, and performs the scaling automatically). + Furthermore, the ct:timetrap/1 function has been + introduced, which makes it possible to set/reset + timetraps during test execution. Also, a ct:sleep/1 + function is now available, which takes the timetrap + parameters into account when calculating the time to + suspend the process.</p> + <p> + Own Id: OTP-8693</p> + </item> + <item> + <p> + A new run_test start option, event_handler_init, has been + added that takes a start argument which gets passed to + the init function of the event handler.</p> + <p> + Own Id: OTP-8694</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.4.7</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index beb3ed3247..8be234d979 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -4,7 +4,7 @@ <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Common Test Reference Manual</title> @@ -75,6 +75,7 @@ <xi:include href="ct_snmp.xml"/> <xi:include href="ct_telnet.xml"/> <xi:include href="unix_telnet.xml"/> + <xi:include href="ct_slave.xml"/> </application> diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/run_test.xml index d9dd22d411..2f0a94afba 100644 --- a/lib/common_test/doc/src/run_test.xml +++ b/lib/common_test/doc/src/run_test.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>2007</year><year>2009</year> + <year>2007</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,62 +13,95 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> - <title>The run_test shell script</title> + <title>The run_test program</title> <prepared>Peter Andersson</prepared> <responsible>Peter Andersson</responsible> <docno></docno> <approved></approved> <checked></checked> - <date>2007-07-04</date> - <rev>PA1</rev> + <date>2010-04-01</date> + <rev>PA2</rev> <file>run_test.xml</file> </header> - <com>run_test</com> - <comsummary>Shell script used for starting - Common Test from the Unix command line. + <com>run_test</com> + <comsummary>Program used for starting Common Test from the + OS command line. </comsummary> <description> - <p>The <c>run_test</c> script is automatically generated as Common - Test is installed (please see the Installation chapter in the Common - Test User's Guide for more information). The script accepts a number - of different start flags. Some flags trigger <c>run_test</c> - to start the Common Test application and pass on data to it. Some - flags start an Erlang node prepared for running Common Test in a - particular mode.</p> - <p><c>run_test</c> also accepts Erlang emulator - flags. These are used when <c>run_test</c> calls <c>erl</c> to - start the Erlang node (making it possible to e.g. add directories to - the code server path, change the cookie on the node, start - additional applications, etc).</p> - <p>If <c>run_test</c> is called without parameters, it prints all valid - start flags to stdout.</p> + <p>The <c>run_test</c> program is automatically installed with Erlang/OTP + and Common Test (please see the Installation chapter in the Common + Test User's Guide for more information). The program accepts a number + of different start flags. Some flags trigger <c>run_test</c> + to start the Common Test application and pass on data to it. Some + flags start an Erlang node prepared for running Common Test in a + particular mode.</p> + + <p>There is an interface function that corresponds to this program, + called <c>ct:run_test/1</c>, for starting Common Test from the Erlang + shell (or an Erlang program). Please see the <c>ct</c> man page for + details.</p> + + <p><c>run_test</c> also accepts Erlang emulator flags. These are used + when <c>run_test</c> calls <c>erl</c> to start the Erlang node + (making it possible to e.g. add directories to the code server path, + change the cookie on the node, start additional applications, etc).</p> + + <p>With the optional flag:</p> + <pre>-erl_args</pre> + <p>it's possible to divide the options on the <c>run_test</c> command line into + two groups, one that Common Test should process (those preceding <c>-erl_args</c>), + and one it should completely ignore and pass on directly to the emulator + (those following <c>-erl_args</c>). Options preceding <c>-erl_args</c> that Common Test + doesn't recognize, also get passed on to the emulator untouched. + By means of <c>-erl_args</c> the user may specify flags with the same name, but + with different destinations, on the <c>run_test</c> command line.</p> + <p>If <c>-pa</c> or <c>-pz</c> flags are specified in the Common Test group of options + (preceding <c>-erl_args</c>), relative directories will be converted to + absolute and re-inserted into the code path by Common Test (to avoid + problems loading user modules when Common Test changes working directory + during test runs). Common Test will however ignore <c>-pa</c> and <c>-pz</c> flags + following <c>-erl_args</c> on the command line. These directories are added + to the code path normally (i.e. on specified form)</p> + + <p>If <c>run_test</c> is called with option:</p> + <pre>-help</pre> + <p>it prints all valid start flags to stdout.</p> </description> + <marker id="run_test"></marker> + <section> <title>Run tests from command line</title> <pre> - run_test [-dir TestDir1 TestDir2 .. TestDirN] | - [-suite Suite1 Suite2 .. SuiteN + run_test [-dir TestDir1 TestDir2 .. TestDirN] | + [-suite Suite1 Suite2 .. SuiteN [[-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]]] [-step [config | keep_inactive]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] [-decrypt_key Key] | [-decrypt_file KeyFile] + [-label Label] [-logdir LogDir] [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] - [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] + [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | + [-event_handler_init EvHandler1 InitArg1 and + EvHandler2 InitArg2 and .. EvHandlerN InitArgN] [-include InclDir1 InclDir2 .. InclDirN] [-no_auto_compile] + [-muliply_timetraps Multiplier] + [-scale_timetraps] [-repeat N [-force_stop]] | [-duration HHMMSS [-force_stop]] | [-until [YYMoMoDD]HHMMSS [-force_stop]] @@ -79,15 +112,22 @@ <pre> run_test -spec TestSpec1 TestSpec2 .. TestSpecN [-config ConfigFile1 ConfigFile2 .. ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] [-decrypt_key Key] | [-decrypt_file KeyFile] + [-label Label] [-logdir LogDir] [-allow_user_terms] [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] - [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] + [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | + [-event_handler_init EvHandler1 InitArg1 and + EvHandler2 InitArg2 and .. EvHandlerN InitArgN] [-include InclDir1 InclDir2 .. InclDirN] [-no_auto_compile] + [-muliply_timetraps Multiplier] + [-scale_timetraps] [-repeat N [-force_stop]] | [-duration HHMMSS [-force_stop]] | [-until [YYMoMoDD]HHMMSS [-force_stop]] @@ -97,12 +137,16 @@ <title>Run tests in web based GUI</title> <pre> run_test -vts [-browser Browser] - [-config ConfigFile1 ConfigFile2 .. ConfigFileN] - [-decrypt_key Key] | [-decrypt_file KeyFile] [-dir TestDir1 TestDir2 .. TestDirN] | [-suite Suite [[-group Group] [-case Case]]] + [-config ConfigFile1 ConfigFile2 .. ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] + [-decrypt_key Key] | [-decrypt_file KeyFile] [-include InclDir1 InclDir2 .. InclDirN] [-no_auto_compile] + [-muliply_timetraps Multiplier] + [-scale_timetraps] [-basic_html]</pre> </section> <section> @@ -113,28 +157,23 @@ <section> <title>Run CT in interactive mode</title> <pre> - run_test -shell + run_test -shell [-config ConfigFile1 ConfigFile2 ... ConfigFileN] + [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 + ConfigString2 and .. and CallbackModuleN ConfigStringN] [-decrypt_key Key] | [-decrypt_file KeyFile]</pre> </section> <section> - <title>Start an Erlang node with a given name</title> - <pre> - run_test -ctname NodeName</pre> - </section> - <section> <title>Start a Common Test Master node</title> <pre> run_test -ctmaster</pre> </section> <section> - <title>See also</title> - <p>Please read the <seealso marker="run_test_chapter">Running Test Suites</seealso> - chapter in the Common Test User's Guide for information about the meaning of the - different start flags.</p> + <title>See also</title> + <p>Please read the <seealso marker="run_test_chapter">Running Test Suites</seealso> + chapter in the Common Test User's Guide for information about the meaning of the + different start flags.</p> </section> </comref> - - diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index d731d18783..1efff25f5b 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,15 +13,15 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> - <title>Running Test Suites</title> + <title>Running Tests</title> <prepared>Peter Andersson, Kenneth Lundin</prepared> <docno></docno> <date></date> @@ -97,25 +97,32 @@ the <c><![CDATA[{auto_compile,false}]]></c> option with <c><![CDATA[ct:run_test/1]]></c>. With automatic compilation disabled, the user is responsible for compiling the test suite modules - (and any help modules) before the test run. Common Test will only verify - that the specified test suites exist before starting the tests.</p> + (and any help modules) before the test run. If the modules can not be loaded + from the local file system during startup of Common Test, the user needs to + pre-load the modules before starting the test. Common Test will only verify + that the specified test suites exist (i.e. that they are, or can be, loaded). + This is useful e.g. if the test suites are transferred and loaded as binaries via + RPC from a remote node.</p> </section> <section> - <title>Running tests from the UNIX command line</title> + <title>Running tests from the OS command line</title> - <p>The script <c>run_test</c> can be used for running tests from - the Unix/Linux command line, e.g. + <p>The <c>run_test</c> program can be used for running tests from + the OS command line, e.g. </p> <list> <item><c><![CDATA[run_test -config <configfilenames> -dir <dirs>]]></c></item> <item><c><![CDATA[run_test -config <configfilenames> -suite <suiteswithfullpath>]]></c> </item> + <item><c><![CDATA[run_test -userconfig <callbackmodulename> <configfilenames> -suite <suiteswithfullpath>]]></c> + </item> <item><c><![CDATA[run_test -config <configfilenames> -suite <suitewithfullpath> -group <groupnames> -case <casenames>]]></c></item> </list> <p>Examples:</p> <p><c>$ run_test -config $CFGS/sys1.cfg $CFGS/sys2.cfg -dir $SYS1_TEST $SYS2_TEST</c></p> + <p><c>$ run_test -userconfig ct_config_xml $CFGS/sys1.xml $CFGS/sys2.xml -dir $SYS1_TEST $SYS2_TEST</c></p> <p><c>$ run_test -suite $SYS1_TEST/setup_SUITE $SYS2_TEST/config_SUITE</c></p> <p><c>$ run_test -suite $SYS1_TEST/setup_SUITE -case start stop</c></p> <p><c>$ run_test -suite $SYS1_TEST/setup_SUITE -group installation -case start stop</c></p> @@ -123,6 +130,8 @@ <p>Other flags that may be used with <c>run_test</c>:</p> <list> <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item> + <item><c><![CDATA[-label <name_of_test_run>]]></c>, associates the test run with a name that gets printed + in the overview HTML log files.</item> <item><c>-refresh_logs</c>, refreshes the top level HTML index files.</item> <item><c>-vts</c>, start web based GUI (see below).</item> <item><c>-shell</c>, start interactive shell mode (see below).</item> @@ -136,8 +145,14 @@ <seealso marker="cover_chapter#cover">Code Coverage Analysis</seealso>).</item> <item><c><![CDATA[-event_handler <event_handlers>]]></c>, to install <seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item> + <item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install + <seealso marker="event_handler_chapter#event_handling">event handlers</seealso> including start arguments.</item> <item><c><![CDATA[-include]]></c>, specifies include directories (see above).</item> <item><c><![CDATA[-no_auto_compile]]></c>, disables the automatic test suite compilation feature (see above).</item> + <item><c><![CDATA[-multiply_timetraps <n>]]></c>, extends <seealso marker="write_test_chapter#timetraps">timetrap + timeout</seealso> values.</item> + <item><c><![CDATA[-scale_timetraps <bool>]]></c>, enables automatic <seealso marker="write_test_chapter#timetraps">timetrap + timeout</seealso> scaling.</item> <item><c><![CDATA[-repeat <n>]]></c>, tells Common Test to repeat the tests n times (see below).</item> <item><c><![CDATA[-duration <time>]]></c>, tells Common Test to repeat the tests for duration of time (see below).</item> <item><c><![CDATA[-until <stop_time>]]></c>, tells Common Test to repeat the tests until stop_time (see below).</item> @@ -165,7 +180,7 @@ the current working directory of the Erlang Runtime System during the test run!</p> </note> - <p>For details on how to generate the <c>run_test</c> script, see the + <p>For more information about the <c>run_test</c> program, see the <seealso marker="install_chapter#general">Installation</seealso> chapter. </p> </section> @@ -174,7 +189,7 @@ <title>Running tests from the Web based GUI</title> <p>The web based GUI, VTS, is started with the <c>run_test</c> - script. From the GUI you can load config files, and select + program. From the GUI you can load config files, and select directories, suites and cases to run. You can also state the config files, directories, suites and cases on the command line when starting the web based GUI. @@ -198,10 +213,11 @@ <p>Example:</p> <p><c><![CDATA[$ run_test -vts -browser 'firefox&']]></c></p> <p>Note that the browser must run as a separate OS process or VTS will hang!</p> - <p>If no specific browser start command is specified, netscape will + <p>If no specific browser start command is specified, Firefox will be the default browser on Unix platforms and Internet Explorer on Windows. - If Common Test fails to start a browser automatically, start your - favourite browser manually instead and type in the URL that Common Test + If Common Test fails to start a browser automatically, or <c>'none'</c> is + specified as the value for -browser (i.e. <c>-browser none</c>), start your + favourite browser manually and type in the URL that Common Test displays in the shell.</p> </section> @@ -211,7 +227,7 @@ <p>Common Test provides an Erlang API for running tests. The main (and most flexible) function for specifying and executing tests is called <c>ct:run_test/1</c>. This function takes the same start parameters as - the <c>run_test</c> script described above, only the flags are instead + the <c>run_test</c> program described above, only the flags are instead given as options in a list of key-value tuples. E.g. a test specified with <c>run_test</c> like:</p> <p><c>$ run_test -suite ./my_SUITE -logdir ./results</c></p> @@ -237,13 +253,14 @@ manually and call <c>ct:install/1</c> to install any configuration data you might need (use <c>[]</c> as argument otherwise), then call <c>ct:start_interactive/0</c> to start Common Test. If you use - the <c>run_test</c> script, you may start the Erlang shell and Common Test + the <c>run_test</c> program, you may start the Erlang shell and Common Test in the same go by using the <c>-shell</c> and, optionally, the <c>-config</c> - flag: + and/or <c>-userconfig</c> flag. Examples: </p> <list> <item><c>run_test -shell</c></item> - <item><c><![CDATA[run_test -shell -config <configfilename>]]></c></item> + <item><c><![CDATA[run_test -shell -config cfg/db.cfg]]></c></item> + <item><c><![CDATA[run_test -shell -userconfig db_login testuser x523qZ]]></c></item> </list> <p>If no config file is given with the <c>run_test</c> command, @@ -268,7 +285,8 @@ 2> ct_telnet:open(unix_telnet). {ok,<0.105.0>} 4> ct_telnet:cmd(unix_telnet, "ls ."). - {ok,["ls .","file1 ...",...]}</pre> + {ok,["ls .","file1 ...",...]} + </pre> <p>Everything that Common Test normally prints in the test case logs, will in the interactive mode be written to a log named @@ -319,54 +337,99 @@ <marker id="test_specifications"></marker> <title>Using test specifications</title> - <p>The most expressive way to specify what to test is to use a so - called test specification. A test specification is a sequence of - Erlang terms. The terms may be declared in a text file or passed - to the test server at runtime as a list (see <c>run_testspec/1</c> - in the manual page for <c>ct</c>). There are two general types - of terms: configuration terms and test specification terms.</p> - <p>With configuration terms it is possible to import configuration - data (similar to <c>run_test -config</c>), specify HTML log - directories (similar to <c>run_test -logdir</c>), give aliases - to test nodes and test directories (to make a specification - easier to read and maintain), enable code coverage analysis - (see the <seealso marker="cover_chapter#cover">Code Coverage - Analysis</seealso> chapter) and specify event_handler plugins + <p>The most flexible way to specify what to test, is to use a so + called test specification. A test specification is a sequence of + Erlang terms. The terms may be declared in a text file or passed + to the test server at runtime as a list + (see <c>run_testspec/1</c> in the manual page + for <c>ct</c>). There are two general types of terms: + configuration terms and test specification terms.</p> + <p>With configuration terms it is possible to e.g. label the test + run (similar to <c>run_test -label</c>), evaluate arbitrary expressions + before starting a test, import configuration + data (similar to + <c>run_test -config/-userconfig</c>), specify HTML log directories (similar + to + <c>run_test -logdir</c>), give aliases to test nodes and test + directories (to make a specification easier to read and + maintain), enable code coverage analysis (see + the <seealso marker="cover_chapter#cover">Code Coverage + Analysis</seealso> chapter) and specify event_handler plugins (see the <seealso marker="event_handler_chapter#event_handling"> - Event Handling</seealso> chapter). There is also a term - for specifying include directories that should be passed on - to the compiler when automatic compilation is performed - (similar to <c>run_test -include</c>, see above).</p> - <p>With test specification terms it is possible to state exactly which - tests should run and in which order. A test term specifies either - one or more suites or one or more test cases. An arbitrary number of test - terms may be declared in sequence. A test term can also specify one or - more test suites or test cases to be skipped. Skipped suites and cases - are not executed and show up in the HTML test log as SKIPPED.</p> - - <note><p>It is not yet possible to specify test case groups in - test specifications. This will be supported in a soon upcoming - release.</p></note> + Event Handling</seealso> chapter). There is also a term for + specifying include directories that should be passed on to the + compiler when automatic compilation is performed (similar + to <c>run_test -include</c>, see above).</p> + <p>With test specification terms it is possible to state exactly + which tests should run and in which order. A test term specifies + either one or more suites, one or more test case groups, or one + or more test cases in a group or suite.</p> + <p>An arbitrary number of test terms may be declared in sequence. + Common Test will compile the terms into one or more tests to be + performed in one resulting test run. Note that a term that + specifies a set of test cases will "swallow" one that only + specifies a subset of these cases. E.g. the result of merging + one term that specifies that all cases in suite S should be + executed, with another term specifying only test case X and Y in + S, is a test of all cases in S. However, if a term specifying + test case X and Y in S is merged with a term specifying case Z + in S, the result is a test of X, Y and Z in S.</p> + <p>A test term can also specify one or more test suites, groups, + or test cases to be skipped. Skipped suites, groups and cases + are not executed and show up in the HTML test log files as + SKIPPED.</p> + <p>When a test case group is specified, the resulting test + executes the + <c>init_per_group</c> function, followed by all test cases and + sub groups (including their configuration functions), and + finally the <c>end_per_group</c> function. Also if particular + test cases in a group are specified, <c>init_per_group</c> + and <c>end_per_group</c> for the group in question are + called. If a group which is defined (in <c>Suite:group/0</c>) to + be a sub group of another group, is specified (or particular test + cases of a sub group are), Common Test will call the configuration + functions for the top level groups as well as for the sub group + in question (making it possible to pass configuration data all + the way from <c>init_per_suite</c> down to the test cases in the + sub group).</p> <p>Below is the test specification syntax. Test specifications can - be used to run tests both in a single test host environment and in - a distributed Common Test environment. Node parameters are only relevant in the - latter (see the chapter about running Common Test in distributed mode for information). - For details on the event_handler term, see the - <seealso marker="event_handler_chapter#event_handling">Event Handling</seealso> - chapter.</p> + be used to run tests both in a single test host environment and + in a distributed Common Test environment (Large Scale + Testing). The node parameters in the init term are only + relevant in the latter (see the + <seealso marker="ct_master_chapter#test_specifications">Large + Scale Testing</seealso> chapter for information). For details on + the event_handler term, see the + <seealso marker="event_handler_chapter#event_handling">Event + Handling</seealso> chapter.</p> <p>Config terms:</p> <pre> {node, NodeAlias, Node}. + + {init, InitOptions}. + {init, [NodeAlias], InitOptions}. + + {label, Label}. + {label, NodeRefs, Label}. + + {multiply_timetraps, N}. + {multiply_timetraps, NodeRefs, N}. + + {scale_timetraps, Bool}. + {scale_timetraps, NodeRefs, Bool}. {cover, CoverSpecFile}. - {cover, NodeRef, CoverSpecFile}. + {cover, NodeRefs, CoverSpecFile}. {include, IncludeDirs}. {include, NodeRefs, IncludeDirs}. {config, ConfigFiles}. {config, NodeRefs, ConfigFiles}. + + {userconfig, {CallbackModule, ConfigStrings}}. + {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}. {alias, DirAlias, Dir}. @@ -383,6 +446,12 @@ {suites, DirRef, Suites}. {suites, NodeRefs, DirRef, Suites}. + {groups, DirRef, Suite, Groups}. + {groups, NodeRefsDirRef, Suite, Groups}. + + {groups, DirRef, Suite, Group, {cases,Cases}}. + {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}. + {cases, DirRef, Suite, Cases}. {cases, NodeRefs, DirRef, Suite, Cases}. @@ -395,9 +464,12 @@ <p>Types:</p> <pre> NodeAlias = atom() + InitOptions = term() Node = node() NodeRef = NodeAlias | Node | master NodeRefs = all_nodes | [NodeRef] | NodeRef + N = integer() + Bool = true | false CoverSpecFile = string() IncludeDirs = string() | [string()] ConfigFiles = string() | [string()] @@ -408,6 +480,9 @@ InitArgs = [term()] DirRef = DirAlias | Dir Suites = atom() | [atom()] | all + Suite = atom() + Groups = atom() | [atom()] | all + Group = atom() Cases = atom() | [atom()] | all Comment = string() | "" </pre> @@ -449,9 +524,14 @@ <item>Secondly, the test for system t2 should run. The included suites are t2B and t2C. Included are also test cases test4, test1 and test7 in suite t2A. Note that the test cases will be executed in the specified order.</item> - <item>Lastly, all suites for systems t3 are to be completely skipped and this + <item>Lastly, all suites for systems t3 are to be completely skipped and this should be explicitly noted in the log files.</item> </list> + <p>It is possible to specify initialization options for nodes defined in the + test specification. Currently, there are options to start the node and/or to + evaluate any function on the node. + See the <seealso marker="ct_master_chapter#ct_slave">Automatic startup of + the test target nodes</seealso> chapter for details.</p> <p>It is possible for the user to provide a test specification that includes (for Common Test) unrecognizable terms. If this is desired, the <c>-allow_user_terms</c> flag should be used when starting tests with diff --git a/lib/common_test/doc/src/test_structure_chapter.xml b/lib/common_test/doc/src/test_structure_chapter.xml index c8628b3a7a..cd38ae0c7c 100644 --- a/lib/common_test/doc/src/test_structure_chapter.xml +++ b/lib/common_test/doc/src/test_structure_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2009</year> + <year>2006</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Test Structure</title> @@ -146,8 +146,8 @@ <tag><em>run_test</em></tag> <item> - The name of an executable Bourne shell script that may be - used on Linux/Unix as an interface for specifying and running + The name of an executable program that may be + used as an interface for specifying and running tests with Common Test. </item> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 212e3d85be..5afec6de6a 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -13,12 +13,12 @@ compliance with the License. You should have received a copy of the Erlang Public License along with this software. If not, it can be retrieved online at http://www.erlang.org/. - + Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License. - + </legalnotice> <title>Writing Test Suites</title> @@ -157,6 +157,15 @@ <c>{skipped,Reason}</c> (where Reason is a user specific term). </p> + <p>The <c>end_per_testcase/2</c> function is called even after a + test case terminates due to a call to <c>ct:abort_current_testcase/1</c>, + or after a timetrap timeout. However, <c>end_per_testcase</c> + will then execute on a different process than the test case + function, and in this situation, <c>end_per_testcase</c> will + not be able to change the reason for test case termination by + returning <c>{fail,Reason}</c>, nor will it be able to save data with + <c>{save_config,Data}</c>.</p> + <p>If <c>init_per_testcase</c> crashes, the test case itself is skipped automatically (so called <em>auto skipped</em>). If <c>init_per_testcase</c> returns a <c>skip</c> tuple, also then will the test case be skipped (so @@ -682,12 +691,33 @@ <c>end_per_suite</c> execute, like test cases, on dedicated Erlang processes. </p> + </section> + <section> + <title>Timetrap timeouts</title> + <marker id="timetraps"></marker> <p>The default time limit for a test case is 30 minutes, unless a - <c>timetrap</c> is specified either by the test case info function - or the <c>suite/0</c> function. - </p> - + <c>timetrap</c> is specified either by the suite info function + or a test case info function. The timetrap timeout value defined + in <c>suite/0</c> is the value that will be used for each test case + in the suite (as well as for the configuration functions + <c>init_per_suite/1</c> and <c>end_per_suite</c>). A timetrap timeout + value set with the test case info function will override the value set + by <c>suite/0</c>, but only for that particular test case.</p> + <p>It is also possible to set/reset a timetrap during test case (or + configuration function) execution. This is done by calling + <c>ct:timetrap/1</c>. This function will cancel the current timetrap + and start a new one.</p> + <p>Timetrap values can be extended with a multiplier value specified at + startup with the <c>multiply_timetraps</c> option. It is also possible + to let Test Server decide to scale up timetrap timeout values + automatically, e.g. if tools such as cover or trace are running during + the test. This feature is disabled by default and can be enabled with + the <c>scale_timetraps</c> start option.</p> + <p>If a test case needs to suspend itself for a time that also gets + multipled by <c>multiply_timetraps</c>, and possibly scaled up if + <c>scale_timetraps</c> is enabled, the function <c>ct:sleep/1</c> + may be called.</p> </section> <section> diff --git a/lib/common_test/priv/Makefile.in b/lib/common_test/priv/Makefile.in index f144847a29..6372bbc8d5 100644 --- a/lib/common_test/priv/Makefile.in +++ b/lib/common_test/priv/Makefile.in @@ -56,8 +56,9 @@ ifneq ($(findstring win32,$(TARGET)),win32) # # Files # -FILES = vts.tool run_test.in -SCRIPTS = install.sh +FILES = +SCRIPTS = +IMAGES = tile1.jpg # # Rules @@ -83,14 +84,12 @@ include $(ERL_TOP)/make/otp_release_targets.mk ifeq ($(XNIX),true) release_spec: opt - $(INSTALL_DIR) $(RELSYSDIR)/priv/bin - $(INSTALL_SCRIPT) $(SCRIPTS) $(RELSYSDIR) - $(INSTALL_DATA) $(FILES) $(RELSYSDIR)/priv + $(INSTALL_DIR) $(RELSYSDIR)/priv + $(INSTALL_DATA) $(FILES) $(IMAGES) $(RELSYSDIR)/priv else release_spec: opt - $(INSTALL_DIR) $(RELSYSDIR)/priv/bin - $(INSTALL_SCRIPT) $(SCRIPTS) $(RELSYSDIR) - $(INSTALL_DATA) $(FILES) $(RELSYSDIR)/priv + $(INSTALL_DIR) $(RELSYSDIR)/priv + $(INSTALL_DATA) $(FILES) $(IMAGES) $(RELSYSDIR)/priv endif release_docs_spec: @@ -105,6 +104,7 @@ else # Files # FILES = vts.tool +IMAGES = tile1.jpg # # Rules @@ -123,8 +123,8 @@ clean: include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt - $(INSTALL_DIR) $(RELSYSDIR)/priv/bin - $(INSTALL_DATA) $(FILES) $(RELSYSDIR)/priv + $(INSTALL_DIR) $(RELSYSDIR)/priv + $(INSTALL_DATA) $(FILES) $(IMAGES) $(RELSYSDIR)/priv release_docs_spec: diff --git a/lib/common_test/priv/tile1.jpg b/lib/common_test/priv/tile1.jpg Binary files differnew file mode 100644 index 0000000000..8749383716 --- /dev/null +++ b/lib/common_test/priv/tile1.jpg diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index e7e2d1275d..027667e6b0 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 2003-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # @@ -63,7 +63,11 @@ MODULES= \ ct_telnet_client \ ct_make \ vts \ - unix_telnet + unix_telnet \ + ct_config \ + ct_config_plain \ + ct_config_xml \ + ct_slave TARGET_MODULES= $(MODULES:%=$(EBIN)/%) diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 7b72932ad4..b42173f412 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -1,19 +1,19 @@ % This is an -*- erlang -*- file. %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% {application, common_test, @@ -42,10 +42,15 @@ ct_testspec, ct_util, unix_telnet, - vts + vts, + ct_config, + ct_config_plain, + ct_config_xml, + ct_slave ]}, {registered, [ct_logs, ct_util_server, + ct_config_server, ct_make_ref, vts, ct_master, diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 8ae041e5b4..8ae175f10d 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -59,18 +59,22 @@ %% Test suite API -export([require/1, require/2, get_config/1, get_config/2, get_config/3, + reload_config/1, log/1, log/2, log/3, print/1, print/2, print/3, pal/1, pal/2, pal/3, fail/1, comment/1, - testcases/2, userdata/2, userdata/3]). + testcases/2, userdata/2, userdata/3, + timetrap/1, sleep/1]). + +%% New API for manipulating with config handlers +-export([add_config/2, remove_config/2]). %% Other interface functions -export([get_status/0, abort_current_testcase/1, encrypt_config_file/2, encrypt_config_file/3, decrypt_config_file/2, decrypt_config_file/3]). - -export([get_target_name/1]). -export([parse_table/1, listenv/1]). @@ -93,7 +97,7 @@ %%% <code>install([{config,["config_node.ctc","config_user.ctc"]}])</code>.</p> %%% %%% <p>Note that this function is automatically run by the -%%% <code>run_test</code> script.</p> +%%% <code>run_test</code> program.</p> install(Opts) -> ct_run:install(Opts). @@ -134,22 +138,30 @@ run(TestDirs) -> %%%----------------------------------------------------------------- %%% @spec run_test(Opts) -> Result %%% Opts = [OptTuples] -%%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} | -%%% {testcase,Cases} | {group,Groups} | {spec,TestSpecs} | -%%% {allow_user_terms,Bool} | {logdir,LogDir} | -%%% {silent_connections,Conns} | {cover,CoverSpecFile} | -%%% {step,StepOpts} | {event_handler,EventHandlers} | {include,InclDirs} | -%%% {auto_compile,Bool} | {repeat,N} | {duration,DurTime} | -%%% {until,StopTime} | {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | +%%% OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} | +%%% {testcase,Cases} | {spec,TestSpecs} | {label,Label} | +%%% {config,CfgFiles} | {userconfig, UserConfig} | +%%% {allow_user_terms,Bool} | {logdir,LogDir} | +%%% {silent_connections,Conns} | {stylesheet,CSSFile} | +%%% {cover,CoverSpecFile} | {step,StepOpts} | +%%% {event_handler,EventHandlers} | {include,InclDirs} | +%%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} | +%%% {repeat,N} | {duration,DurTime} | {until,StopTime} | +%%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | %%% {refresh_logs,LogDir} | {basic_html,Bool} -%%% CfgFiles = [string()] | string() %%% TestDirs = [string()] | string() %%% Suites = [string()] | string() %%% Cases = [atom()] | atom() %%% Groups = [atom()] | atom() %%% TestSpecs = [string()] | string() +%%% Label = string() | atom() +%%% CfgFiles = [string()] | string() +%%% UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings} +%%% CallbackMod = atom() +%%% CfgStrings = [string()] | string() %%% LogDir = string() %%% Conns = all | [atom()] +%%% CSSFile = string() %%% CoverSpecFile = string() %%% StepOpts = [StepOpt] | [] %%% StepOpt = config | keep_inactive @@ -157,6 +169,7 @@ run(TestDirs) -> %%% EH = atom() | {atom(),InitArgs} | {[atom()],InitArgs} %%% InitArgs = [term()] %%% InclDirs = [string()] | string() +%%% M = integer() %%% N = integer() %%% DurTime = string(HHMMSS) %%% StopTime = string(YYMoMoDDHHMMSS) | string(HHMMSS) @@ -165,11 +178,12 @@ run(TestDirs) -> %%% DecryptFile = string() %%% Result = [TestResult] | {error,Reason} %%% @doc Run tests as specified by the combination of options in <code>Opts</code>. -%%% The options are the same as those used with the <code>run_test</code> script. +%%% The options are the same as those used with the +%%% <seealso marker="run_test#run_test"><code>run_test</code></seealso> program. %%% Note that here a <code>TestDir</code> can be used to point out the path to %%% a <code>Suite</code>. Note also that the option <code>testcase</code> %%% corresponds to the <code>-case</code> option in the <code>run_test</code> -%%% script. Configuration files specified in <code>Opts</code> will be +%%% program. Configuration files specified in <code>Opts</code> will be %%% installed automatically at startup. run_test(Opts) -> ct_run:run_test(Opts). @@ -211,7 +225,7 @@ step(TestDir,Suite,Case,Opts) -> %%% %%% <p>From this mode all test case support functions can be executed %%% directly from the erlang shell. The interactive mode can also be -%%% started from the unix command line with <code>run_test -shell +%%% started from the OS command line with <code>run_test -shell %%% [-config File...]</code>.</p> %%% %%% <p>If any functions using "required config data" (e.g. telnet or @@ -269,7 +283,7 @@ stop_interactive() -> %%% @see get_config/2 %%% @see get_config/3 require(Required) -> - ct_util:require(Required). + ct_config:require(Required). %%%----------------------------------------------------------------- %%% @spec require(Name,Required) -> ok | {error,Reason} @@ -304,19 +318,19 @@ require(Required) -> %%% @see get_config/2 %%% @see get_config/3 require(Name,Required) -> - ct_util:require(Name,Required). + ct_config:require(Name,Required). %%%----------------------------------------------------------------- %%% @spec get_config(Required) -> Value %%% @equiv get_config(Required,undefined,[]) get_config(Required) -> - ct_util:get_config(Required,undefined,[]). + ct_config:get_config(Required,undefined,[]). %%%----------------------------------------------------------------- %%% @spec get_config(Required,Default) -> Value %%% @equiv get_config(Required,Default,[]) get_config(Required,Default) -> - ct_util:get_config(Required,Default,[]). + ct_config:get_config(Required,Default,[]). %%%----------------------------------------------------------------- %%% @spec get_config(Required,Default,Opts) -> ValueOrElement @@ -375,7 +389,26 @@ get_config(Required,Default) -> %%% @see require/1 %%% @see require/2 get_config(Required,Default,Opts) -> - ct_util:get_config(Required,Default,Opts). + ct_config:get_config(Required,Default,Opts). + +%%%----------------------------------------------------------------- +%%% @spec reload_config(Required) -> ValueOrElement +%%% Required = KeyOrName | {KeyOrName,SubKey} +%%% KeyOrName = atom() +%%% SubKey = atom() +%%% ValueOrElement = term() +%%% +%%% @doc Reload config file which contains specified configuration key. +%%% +%%% <p>This function performs updating of the configuration data from which the +%%% given configuration variable was read, and returns the (possibly) new +%%% value of this variable.</p> +%%% <p>Note that if some variables were present in the configuration but are not loaded +%%% using this function, they will be removed from the configuration table together +%%% with their aliases.</p> +%%% +reload_config(Required)-> + ct_config:reload_config(Required). %%%----------------------------------------------------------------- %%% @spec log(Format) -> ok @@ -734,7 +767,7 @@ abort_current_testcase(Reason) -> %%% <p>See the <code>crypto</code> application for details on DES3 %%% encryption/decryption.</p> encrypt_config_file(SrcFileName, EncryptFileName) -> - ct_util:encrypt_config_file(SrcFileName, EncryptFileName). + ct_config:encrypt_config_file(SrcFileName, EncryptFileName). %%%----------------------------------------------------------------- %%% @spec encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> @@ -754,7 +787,7 @@ encrypt_config_file(SrcFileName, EncryptFileName) -> %%% <p>See the <code>crypto</code> application for details on DES3 %%% encryption/decryption.</p> encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> - ct_util:encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile). + ct_config:encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile). %%%----------------------------------------------------------------- %%% @spec decrypt_config_file(EncryptFileName, TargetFileName) -> @@ -770,7 +803,7 @@ encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> %%% <code>.ct_config.crypt</code> in the current directory, or the %%% home directory of the user (it is searched for in that order).</p> decrypt_config_file(EncryptFileName, TargetFileName) -> - ct_util:decrypt_config_file(EncryptFileName, TargetFileName). + ct_config:decrypt_config_file(EncryptFileName, TargetFileName). %%%----------------------------------------------------------------- %%% @spec decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) -> @@ -785,5 +818,65 @@ decrypt_config_file(EncryptFileName, TargetFileName) -> %%% file contents is saved in the target file. The key must have the %%% the same value as that used for encryption.</p> decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) -> - ct_util:decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile). + ct_config:decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile). + + +%%%----------------------------------------------------------------- +%%% @spec add_config(Callback, Config) -> ok | {error, Reason} +%%% Callback = atom() +%%% Config = string() +%%% Reason = term() +%%% +%%% @doc <p>This function loads configuration variables using the +%%% given callback module and configuration string. Callback module +%%% should be either loaded or present in the code part. Loaded +%%% configuration variables can later be removed using +%%% <code>remove_config/2</code> function.</p> +add_config(Callback, Config)-> + ct_config:add_config(Callback, Config). +%%%----------------------------------------------------------------- +%%% @spec remove_config(Callback, Config) -> ok +%%% Callback = atom() +%%% Config = string() +%%% Reason = term() +%%% +%%% @doc <p>This function removes configuration variables (together with +%%% their aliases) which were loaded with specified callback module and +%%% configuration string.</p> +remove_config(Callback, Config) -> + ct_config:remove_config(Callback, Config). + +%%%----------------------------------------------------------------- +%%% @spec timetrap(Time) -> ok +%%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity +%%% Hours = integer() +%%% Mins = integer() +%%% Secs = integer() +%%% Millisecs = integer() | float() +%%% +%%% @doc <p>Use this function to set a new timetrap for the running test case.</p> +timetrap(Time) -> + test_server:timetrap(Time). + +%%%----------------------------------------------------------------- +%%% @spec sleep(Time) -> ok +%%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity +%%% Hours = integer() +%%% Mins = integer() +%%% Secs = integer() +%%% Millisecs = integer() | float() +%%% +%%% @doc <p>This function, similar to <c>timer:sleep/1</c>, suspends the test +%%% case for specified time. However, this function also multiplies +%%% <c>Time</c> with the 'multiply_timetraps' value (if set) and under +%%% certain circumstances also scales up the time automatically +%%% if 'scale_timetraps' is set to true (default is false).</p> +sleep({hours,Hs}) -> + sleep(trunc(Hs * 1000 * 60 * 60)); +sleep({minutes,Ms}) -> + sleep(trunc(Ms * 1000 * 60)); +sleep({seconds,Ss}) -> + sleep(trunc(Ss * 1000)); +sleep(Time) -> + test_server:adjusted_sleep(Time). diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl new file mode 100644 index 0000000000..dc6fcc66e5 --- /dev/null +++ b/lib/common_test/src/ct_config.erl @@ -0,0 +1,786 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%%---------------------------------------------------------------------- +%% File : ct_config.erl +%% Description : CT module for reading and manipulating of configuration +%% data +%% +%% Created : 15 February 2010 +%%---------------------------------------------------------------------- +-module(ct_config). + +-export([start/1, stop/0]). + +-export([read_config_files/1, + get_config_file_list/1]). + +-export([require/1, require/2]). + +-export([get_config/1, get_config/2, get_config/3, + get_all_config/0]). + +-export([set_default_config/2, set_default_config/3]). + +-export([delete_default_config/1]). + +-export([reload_config/1, update_config/2]). + +-export([release_allocated/0]). + +-export([encrypt_config_file/2, encrypt_config_file/3, + decrypt_config_file/2, decrypt_config_file/3, + get_crypt_key_from_file/0, get_crypt_key_from_file/1]). + +-export([get_ref_from_name/1, get_name_from_ref/1, get_key_from_name/1]). + +-export([check_config_files/1, prepare_config_list/1]). + +-export([add_config/2, remove_config/2]). + +-include("ct_util.hrl"). + +-define(cryptfile, ".ct_config.crypt"). + +-record(ct_conf,{key,value,handler,config,ref,name='_UNDEF',default=false}). + +start(Mode) -> + case whereis(ct_config_server) of + undefined -> + Me = self(), + Pid = spawn_link(fun() -> do_start(Me) end), + receive + {Pid,started} -> Pid; + {Pid,Error} -> exit(Error) + end; + Pid -> + case ct_util:get_mode() of + interactive when Mode==interactive -> + Pid; + interactive -> + {error,interactive_mode}; + _OtherMode -> + Pid + end + end. + +do_start(Parent) -> + process_flag(trap_exit,true), + register(ct_config_server,self()), + ct_util:create_table(?attr_table,bag,#ct_conf.key), + {ok,StartDir} = file:get_cwd(), + Opts = case ct_util:read_opts() of + {ok,Opts1} -> + Opts1; + Error -> + Parent ! {self(),Error}, + exit(Error) + end, + case read_config_files(Opts) of + ok -> + Parent ! {self(),started}, + loop(StartDir); + ReadError -> + Parent ! {self(),ReadError}, + exit(ReadError) + end. + +stop() -> + case whereis(ct_config_server) of + undefined -> ok; + _ -> call({stop}) + end. + +call(Msg) -> + MRef = erlang:monitor(process, whereis(ct_config_server)), + Ref = make_ref(), + ct_config_server ! {Msg,{self(),Ref}}, + receive + {Ref, Result} -> + erlang:demonitor(MRef, [flush]), + Result; + {'DOWN',MRef,process,_,Reason} -> + {error,{ct_util_server_down,Reason}} + end. + +return({To,Ref},Result) -> + To ! {Ref, Result}. + +loop(StartDir) -> + receive + {{require,Name,Tag,SubTags},From} -> + Result = do_require(Name,Tag,SubTags), + return(From,Result), + loop(StartDir); + {{set_default_config,{Config,Scope}},From} -> + set_config(Config,{true,Scope}), + return(From,ok), + loop(StartDir); + {{set_default_config,{Name,Config,Scope}},From} -> + set_config(Name,Config,{true,Scope}), + return(From,ok), + loop(StartDir); + {{delete_default_config,Scope},From} -> + delete_config({true,Scope}), + return(From,ok), + loop(StartDir); + {{update_config,{Name,NewConfig}},From} -> + update_conf(Name,NewConfig), + return(From,ok), + loop(StartDir); + {{reload_config, KeyOrName},From}-> + NewValue = reload_conf(KeyOrName), + return(From, NewValue), + loop(StartDir); + {{stop},From} -> + ets:delete(?attr_table), + file:set_cwd(StartDir), + return(From,ok) + end. + +set_default_config(NewConfig, Scope) -> + call({set_default_config, {NewConfig, Scope}}). + +set_default_config(Name, NewConfig, Scope) -> + call({set_default_config, {Name, NewConfig, Scope}}). + +delete_default_config(Scope) -> + call({delete_default_config, Scope}). + +update_config(Name, Config) -> + call({update_config, {Name, Config}}). + +reload_config(KeyOrName) -> + call({reload_config, KeyOrName}). + +process_default_configs(Opts) -> + case lists:keysearch(config, 1, Opts) of + {value,{_,Files=[File|_]}} when is_list(File) -> + Files; + {value,{_,File=[C|_]}} when is_integer(C) -> + [File]; + {value,{_,[]}} -> + []; + false -> + [] + end. + +process_user_configs(Opts, Acc) -> + case lists:keytake(userconfig, 1, Opts) of + false -> + lists:reverse(Acc); + {value, {userconfig, Config=[{_,_}|_]}, NewOpts} -> + Acc1 = lists:map(fun({_Callback, []}=Cfg) -> + Cfg; + ({Callback, Files=[File|_]}) when is_list(File) -> + {Callback, Files}; + ({Callback, File=[C|_]}) when is_integer(C) -> + {Callback, [File]} + end, Config), + process_user_configs(NewOpts, lists:reverse(Acc1)++Acc); + {value, {userconfig, {Callback, []}}, NewOpts} -> + process_user_configs(NewOpts, [{Callback, []} | Acc]); + {value, {userconfig, {Callback, Files=[File|_]}}, NewOpts} when is_list(File) -> + process_user_configs(NewOpts, [{Callback, Files} | Acc]); + {value, {userconfig, {Callback, File=[C|_]}}, NewOpts} when is_integer(C) -> + process_user_configs(NewOpts, [{Callback, [File]} | Acc]) + end. + +get_config_file_list(Opts) -> + DefaultConfigs = process_default_configs(Opts), + CfgFiles = + if + DefaultConfigs == []-> + []; + true-> + [{?ct_config_txt, DefaultConfigs}] + end ++ + process_user_configs(Opts, []), + CfgFiles. + +read_config_files(Opts) -> + AddCallback = fun(CallBack, []) -> + [{CallBack, []}]; + (CallBack, [F|_]=Files) when is_integer(F) -> + [{CallBack, Files}]; + (CallBack, [F|_]=Files) when is_list(F) -> + lists:map(fun(X) -> {CallBack, X} end, Files) + end, + ConfigFiles = case lists:keyfind(config, 1, Opts) of + {config, ConfigLists}-> + lists:foldr(fun({Callback,Files}, Acc) -> + AddCallback(Callback,Files) ++ Acc + end, + [], + ConfigLists); + false-> + [] + end, + read_config_files_int(ConfigFiles, fun store_config/3). + +read_config_files_int([{Callback, File}|Files], FunToSave) -> + case Callback:read_config(File) of + {ok, Config} -> + FunToSave(Config, Callback, File), + read_config_files_int(Files, FunToSave); + {error, ErrorName, ErrorDetail}-> + {user_error, {ErrorName, File, ErrorDetail}} + end; +read_config_files_int([], _FunToSave) -> + ok. + +store_config(Config, Callback, File) -> + [ets:insert(?attr_table, + #ct_conf{key=Key, + value=Val, + handler=Callback, + config=File, + ref=ct_util:ct_make_ref(), + default=false}) || + {Key,Val} <- Config]. + +keyfindall(Key, Pos, List) -> + [E || E <- List, element(Pos, E) =:= Key]. + +rewrite_config(Config, Callback, File) -> + OldRows = ets:match_object(?attr_table, + #ct_conf{handler=Callback, + config=File,_='_'}), + ets:match_delete(?attr_table, + #ct_conf{handler=Callback, + config=File,_='_'}), + Updater = fun({Key, Value}) -> + case keyfindall(Key, #ct_conf.key, OldRows) of + []-> + ets:insert(?attr_table, + #ct_conf{key=Key, + value=Value, + handler=Callback, + config=File, + ref=ct_util:ct_make_ref()}); + RowsToUpdate -> + Inserter = fun(Row) -> + ets:insert(?attr_table, + Row#ct_conf{value=Value, + ref=ct_util:ct_make_ref()}) + end, + lists:foreach(Inserter, RowsToUpdate) + end + end, + [Updater({Key, Value})||{Key, Value}<-Config]. + +set_config(Config,Default) -> + set_config('_UNDEF',Config,Default). + +set_config(Name,Config,Default) -> + [ets:insert(?attr_table, + #ct_conf{key=Key,value=Val,ref=ct_util:ct_make_ref(), + name=Name,default=Default}) || + {Key,Val} <- Config]. + +get_config(KeyOrName) -> + get_config(KeyOrName,undefined,[]). + +get_config(KeyOrName,Default) -> + get_config(KeyOrName,Default,[]). + +get_config(KeyOrName,Default,Opts) when is_atom(KeyOrName) -> + case lookup_config(KeyOrName) of + [] -> + Default; + [{_Ref,Val}|_] = Vals -> + case {lists:member(all,Opts),lists:member(element,Opts)} of + {true,true} -> + [{KeyOrName,V} || {_R,V} <- lists:sort(Vals)]; + {true,false} -> + [V || {_R,V} <- lists:sort(Vals)]; + {false,true} -> + {KeyOrName,Val}; + {false,false} -> + Val + end + end; + +get_config({KeyOrName,SubKey},Default,Opts) -> + case lookup_config(KeyOrName) of + [] -> + Default; + Vals -> + Vals1 = case [Val || {_Ref,Val} <- lists:sort(Vals)] of + Result=[L|_] when is_list(L) -> + case L of + [{_,_}|_] -> + Result; + _ -> + [] + end; + _ -> + [] + end, + case get_subconfig([SubKey],Vals1,[],Opts) of + {ok,[{_,SubVal}|_]=SubVals} -> + case {lists:member(all,Opts),lists:member(element,Opts)} of + {true,true} -> + [{{KeyOrName,SubKey},Val} || {_,Val} <- SubVals]; + {true,false} -> + [Val || {_SubKey,Val} <- SubVals]; + {false,true} -> + {{KeyOrName,SubKey},SubVal}; + {false,false} -> + SubVal + end; + _ -> + Default + end + end. + +get_subconfig(SubKeys,Values) -> + get_subconfig(SubKeys,Values,[],[]). + +get_subconfig(SubKeys,[Value|Rest],Mapped,Opts) -> + case do_get_config(SubKeys,Value,[]) of + {ok,SubMapped} -> + case lists:member(all,Opts) of + true -> + get_subconfig(SubKeys,Rest,Mapped++SubMapped,Opts); + false -> + {ok,SubMapped} + end; + _Error -> + get_subconfig(SubKeys,Rest,Mapped,Opts) + end; +get_subconfig(SubKeys,[],[],_) -> + {error,{not_available,SubKeys}}; +get_subconfig(_SubKeys,[],Mapped,_) -> + {ok,Mapped}. + +do_get_config([Key|Required],Available,Mapped) -> + case lists:keysearch(Key,1,Available) of + {value,{Key,Value}} -> + NewAvailable = lists:keydelete(Key,1,Available), + NewMapped = [{Key,Value}|Mapped], + do_get_config(Required,NewAvailable,NewMapped); + false -> + {error,{not_available,Key}} + end; +do_get_config([],_Available,Mapped) -> + {ok,lists:reverse(Mapped)}. + +get_all_config() -> + ets:select(?attr_table,[{#ct_conf{name='$1',key='$2',value='$3', + default='$4',_='_'}, + [], + [{{'$1','$2','$3','$4'}}]}]). + +lookup_config(KeyOrName) -> + case lookup_name(KeyOrName) of + [] -> + lookup_key(KeyOrName); + Values -> + Values + end. + +lookup_name(Name) -> + ets:select(?attr_table,[{#ct_conf{ref='$1',value='$2',name=Name,_='_'}, + [], + [{{'$1','$2'}}]}]). +lookup_key(Key) -> + ets:select(?attr_table,[{#ct_conf{key=Key,ref='$1',value='$2',name='_UNDEF',_='_'}, + [], + [{{'$1','$2'}}]}]). + +lookup_handler_for_config({Key, _Subkey}) -> + lookup_handler_for_config(Key); +lookup_handler_for_config(KeyOrName) -> + case lookup_handler_for_name(KeyOrName) of + [] -> + lookup_handler_for_key(KeyOrName); + Values -> + Values + end. + +lookup_handler_for_name(Name) -> + ets:select(?attr_table,[{#ct_conf{handler='$1',config='$2',name=Name,_='_'}, + [], + [{{'$1','$2'}}]}]). + +lookup_handler_for_key(Key) -> + ets:select(?attr_table,[{#ct_conf{handler='$1',config='$2',key=Key,_='_'}, + [], + [{{'$1','$2'}}]}]). + + +update_conf(Name, NewConfig) -> + Old = ets:select(?attr_table,[{#ct_conf{name=Name,_='_'},[],['$_']}]), + lists:foreach(fun(OldElem) -> + NewElem = OldElem#ct_conf{value=NewConfig}, + ets:delete_object(?attr_table, OldElem), + ets:insert(?attr_table, NewElem) + end, Old), + ok. + +reload_conf(KeyOrName) -> + case lookup_handler_for_config(KeyOrName) of + []-> + undefined; + HandlerList-> + HandlerList2 = lists:usort(HandlerList), + read_config_files_int(HandlerList2, fun rewrite_config/3), + get_config(KeyOrName) + end. + +release_allocated() -> + Allocated = ets:select(?attr_table,[{#ct_conf{name='$1',_='_'}, + [{'=/=','$1','_UNDEF'}], + ['$_']}]), + release_allocated(Allocated). +release_allocated([H|T]) -> + ets:delete_object(?attr_table,H), + ets:insert(?attr_table,H#ct_conf{name='_UNDEF'}), + release_allocated(T); +release_allocated([]) -> + ok. + +allocate(Name,Key,SubKeys) -> + case ets:match_object(?attr_table,#ct_conf{key=Key,name='_UNDEF',_='_'}) of + [] -> + {error,{not_available,Key}}; + Available -> + case allocate_subconfig(Name,SubKeys,Available,false) of + ok -> + ok; + Error -> + Error + end + end. + +allocate_subconfig(Name,SubKeys,[C=#ct_conf{value=Value}|Rest],Found) -> + case do_get_config(SubKeys,Value,[]) of + {ok,_SubMapped} -> + ets:insert(?attr_table,C#ct_conf{name=Name}), + allocate_subconfig(Name,SubKeys,Rest,true); + _Error -> + allocate_subconfig(Name,SubKeys,Rest,Found) + end; +allocate_subconfig(_Name,_SubKeys,[],true) -> + ok; +allocate_subconfig(_Name,SubKeys,[],false) -> + {error,{not_available,SubKeys}}. + +delete_config(Default) -> + ets:match_delete(?attr_table,#ct_conf{default=Default,_='_'}), + ok. + +require(Key) when is_atom(Key) -> + require({Key,[]}); +require({Key,SubKeys}) when is_atom(Key) -> + allocate('_UNDEF',Key,to_list(SubKeys)); +require(Key) -> + {error,{invalid,Key}}. + +require(Name,Key) when is_atom(Key) -> + require(Name,{Key,[]}); +require(Name,{Key,SubKeys}) when is_atom(Name), is_atom(Key) -> + call({require,Name,Key,to_list(SubKeys)}); +require(Name,Keys) -> + {error,{invalid,{Name,Keys}}}. + +to_list(X) when is_list(X) -> X; +to_list(X) -> [X]. + +do_require(Name,Key,SubKeys) when is_list(SubKeys) -> + case get_key_from_name(Name) of + {error,_} -> + allocate(Name,Key,SubKeys); + {ok,Key} -> + %% already allocated - check that it has all required subkeys + Vals = [Val || {_Ref,Val} <- lookup_name(Name)], + case get_subconfig(SubKeys,Vals) of + {ok,_SubMapped} -> + ok; + Error -> + Error + end; + {ok,OtherKey} -> + {error,{name_in_use,Name,OtherKey}} + end. + +encrypt_config_file(SrcFileName, EncryptFileName) -> + case get_crypt_key_from_file() of + {error,_} = E -> + E; + Key -> + encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) + end. + +get_ref_from_name(Name) -> + case ets:select(?attr_table,[{#ct_conf{name=Name,ref='$1',_='_'}, + [], + ['$1']}]) of + [Ref] -> + {ok,Ref}; + _ -> + {error,{no_such_name,Name}} + end. + +get_name_from_ref(Ref) -> + case ets:select(?attr_table,[{#ct_conf{name='$1',ref=Ref,_='_'}, + [], + ['$1']}]) of + [Name] -> + {ok,Name}; + _ -> + {error,{no_such_ref,Ref}} + end. + +get_key_from_name(Name) -> + case ets:select(?attr_table,[{#ct_conf{name=Name,key='$1',_='_'}, + [], + ['$1']}]) of + [Key|_] -> + {ok,Key}; + _ -> + {error,{no_such_name,Name}} + end. + +encrypt_config_file(SrcFileName, EncryptFileName, {file,KeyFile}) -> + case get_crypt_key_from_file(KeyFile) of + {error,_} = E -> + E; + Key -> + encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) + end; + +encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) -> + crypto:start(), + {K1,K2,K3,IVec} = make_crypto_key(Key), + case file:read_file(SrcFileName) of + {ok,Bin0} -> + Bin1 = term_to_binary({SrcFileName,Bin0}), + Bin2 = case byte_size(Bin1) rem 8 of + 0 -> Bin1; + N -> list_to_binary([Bin1,random_bytes(8-N)]) + end, + EncBin = crypto:des3_cbc_encrypt(K1, K2, K3, IVec, Bin2), + case file:write_file(EncryptFileName, EncBin) of + ok -> + io:format("~s --(encrypt)--> ~s~n", + [SrcFileName,EncryptFileName]), + ok; + {error,Reason} -> + {error,{Reason,EncryptFileName}} + end; + {error,Reason} -> + {error,{Reason,SrcFileName}} + end. + +decrypt_config_file(EncryptFileName, TargetFileName) -> + case get_crypt_key_from_file() of + {error,_} = E -> + E; + Key -> + decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) + end. + +decrypt_config_file(EncryptFileName, TargetFileName, {file,KeyFile}) -> + case get_crypt_key_from_file(KeyFile) of + {error,_} = E -> + E; + Key -> + decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) + end; + +decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) -> + crypto:start(), + {K1,K2,K3,IVec} = make_crypto_key(Key), + case file:read_file(EncryptFileName) of + {ok,Bin} -> + DecBin = crypto:des3_cbc_decrypt(K1, K2, K3, IVec, Bin), + case catch binary_to_term(DecBin) of + {'EXIT',_} -> + {error,bad_file}; + {_SrcFile,SrcBin} -> + case TargetFileName of + undefined -> + {ok,SrcBin}; + _ -> + case file:write_file(TargetFileName, SrcBin) of + ok -> + io:format("~s --(decrypt)--> ~s~n", + [EncryptFileName,TargetFileName]), + ok; + {error,Reason} -> + {error,{Reason,TargetFileName}} + end + end + end; + {error,Reason} -> + {error,{Reason,EncryptFileName}} + end. + +get_crypt_key_from_file(File) -> + case file:read_file(File) of + {ok,Bin} -> + case catch string:tokens(binary_to_list(Bin), [$\n,$\r]) of + [Key] -> + Key; + _ -> + {error,{bad_crypt_file,File}} + end; + {error,Reason} -> + {error,{Reason,File}} + end. + +get_crypt_key_from_file() -> + CwdFile = filename:join(".",?cryptfile), + {Result,FullName} = + case file:read_file(CwdFile) of + {ok,Bin} -> + {Bin,CwdFile}; + _ -> + case init:get_argument(home) of + {ok,[[Home]]} -> + HomeFile = filename:join(Home,?cryptfile), + case file:read_file(HomeFile) of + {ok,Bin} -> + {Bin,HomeFile}; + _ -> + {{error,no_crypt_file},noent} + end; + _ -> + {{error,no_crypt_file},noent} + end + end, + case FullName of + noent -> + Result; + _ -> + case catch string:tokens(binary_to_list(Result), [$\n,$\r]) of + [Key] -> + io:format("~nCrypt key file: ~s~n", [FullName]), + Key; + _ -> + {error,{bad_crypt_file,FullName}} + end + end. + +make_crypto_key(String) -> + <<K1:8/binary,K2:8/binary>> = First = erlang:md5(String), + <<K3:8/binary,IVec:8/binary>> = erlang:md5([First|lists:reverse(String)]), + {K1,K2,K3,IVec}. + +random_bytes(N) -> + {A,B,C} = now(), + random:seed(A, B, C), + random_bytes_1(N, []). + +random_bytes_1(0, Acc) -> Acc; +random_bytes_1(N, Acc) -> random_bytes_1(N-1, [random:uniform(255)|Acc]). + +check_callback_load(Callback) -> + case code:is_loaded(Callback) of + {file, _Filename}-> + check_exports(Callback); + false-> + case code:load_file(Callback) of + {module, Callback}-> + check_exports(Callback); + {error, Error}-> + {error, Error} + end + end. + +check_exports(Callback) -> + Fs = Callback:module_info(exports), + case {lists:member({check_parameter,1},Fs), + lists:member({read_config,1},Fs)} of + {true, true} -> + {ok, Callback}; + _ -> + {error, missing_callback_functions} + end. + +check_config_files(Configs) -> + ConfigChecker = fun + ({Callback, [F|_R]=Files}) -> + case check_callback_load(Callback) of + {ok, Callback} -> + if is_integer(F) -> + Callback:check_parameter(Files); + is_list(F) -> + lists:map(fun(File) -> + Callback:check_parameter(File) + end, + Files) + end; + {error, Why}-> + {error, {callback, {Callback,Why}}} + end; + ({Callback, []}) -> + case check_callback_load(Callback) of + {ok, Callback}-> + Callback:check_parameter([]); + {error, Why}-> + {error, {callback, {Callback,Why}}} + end + end, + lists:keysearch(error, 1, lists:flatten(lists:map(ConfigChecker, Configs))). + +prepare_user_configs([ConfigString|UserConfigs], Acc, new) -> + prepare_user_configs(UserConfigs, + [{list_to_atom(ConfigString), []}|Acc], + cur); +prepare_user_configs(["and"|UserConfigs], Acc, _) -> + prepare_user_configs(UserConfigs, Acc, new); +prepare_user_configs([ConfigString|UserConfigs], [{LastMod, LastList}|Acc], cur) -> + prepare_user_configs(UserConfigs, + [{LastMod, [ConfigString|LastList]}|Acc], + cur); +prepare_user_configs([], Acc, _) -> + Acc. + +prepare_config_list(Args) -> + ConfigFiles = case lists:keysearch(ct_config, 1, Args) of + {value,{ct_config,Files}}-> + [{?ct_config_txt,[filename:absname(F) || F <- Files]}]; + false-> + [] + end, + UserConfigs = case lists:keysearch(userconfig, 1, Args) of + {value,{userconfig,UserConfigFiles}}-> + prepare_user_configs(UserConfigFiles, [], new); + false-> + [] + end, + ConfigFiles ++ UserConfigs. + +% TODO: add logging of the loaded configuration file to the CT FW log!!! +add_config(Callback, []) -> + read_config_files_int([{Callback, []}], fun store_config/3); +add_config(Callback, [File|_Files]=Config) when is_list(File) -> + lists:foreach(fun(CfgStr) -> + read_config_files_int([{Callback, CfgStr}], fun store_config/3) end, + Config); +add_config(Callback, [C|_]=Config) when is_integer(C) -> + read_config_files_int([{Callback, Config}], fun store_config/3), + ok. + +remove_config(Callback, Config) -> + ets:match_delete(?attr_table, + #ct_conf{handler=Callback, + config=Config,_='_'}), + ok. diff --git a/lib/common_test/src/ct_config_plain.erl b/lib/common_test/src/ct_config_plain.erl new file mode 100644 index 0000000000..3fbc8af9fb --- /dev/null +++ b/lib/common_test/src/ct_config_plain.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%%---------------------------------------------------------------------- +%% File : ct_config_plain.erl +%% Description : CT callback module for reading configs from text files +%% +%% Created : 15 February 2010 +%%---------------------------------------------------------------------- +-module(ct_config_plain). +-export([read_config/1, check_parameter/1]). + +read_config(ConfigFile) -> + case file:consult(ConfigFile) of + {ok,Config} -> + {ok, Config}; + {error,enoent} -> + {error, config_file_error, enoent}; + {error,Reason} -> + Key = + case application:get_env(common_test, decrypt) of + {ok,KeyOrFile} -> + case KeyOrFile of + {key,K} -> + K; + {file,F} -> + ct_config:get_crypt_key_from_file(F) + end; + _ -> + ct_config:get_crypt_key_from_file() + end, + case Key of + {error,no_crypt_file} -> + {error, config_file_error, Reason}; + {error,CryptError} -> + {error, decrypt_file_error, CryptError}; + _ when is_list(Key) -> + case ct_config:decrypt_config_file(ConfigFile, undefined, {key,Key}) of + {ok,CfgBin} -> + case read_config_terms(CfgBin) of + {error,ReadFail} -> + {error, config_file_error, ReadFail}; + Config -> + {ok, Config} + end; + {error,DecryptFail} -> + {error, decrypt_config_error, DecryptFail} + end; + _ -> + {error, bad_decrypt_key, Key} + end + end. + +% check if config file exists +check_parameter(File)-> + case filelib:is_file(File) of + true-> + {ok, {file, File}}; + false-> + {error, {nofile, File}} + end. + +read_config_terms(Bin) when is_binary(Bin) -> + case catch binary_to_list(Bin) of + {'EXIT',_} -> + {error,invalid_textfile}; + Lines -> + read_config_terms(Lines) + end; +read_config_terms(Lines) when is_list(Lines) -> + read_config_terms1(erl_scan:tokens([], Lines, 0), 1, [], []). + +read_config_terms1({done,{ok,Ts,EL},Rest}, L, Terms, _) -> + case erl_parse:parse_term(Ts) of + {ok,Term} when Rest == [] -> + lists:reverse([Term|Terms]); + {ok,Term} -> + read_config_terms1(erl_scan:tokens([], Rest, 0), + EL+1, [Term|Terms], Rest); + _ -> + {error,{bad_term,{L,EL}}} + end; +read_config_terms1({done,{eof,_},_}, _, Terms, Rest) when Rest == [] -> + lists:reverse(Terms); +read_config_terms1({done,{eof,EL},_}, L, _, _) -> + {error,{bad_term,{L,EL}}}; +read_config_terms1({done,{error,Info,EL},_}, L, _, _) -> + {error,{Info,{L,EL}}}; +read_config_terms1({more,_}, L, Terms, Rest) -> + case string:tokens(Rest, [$\n,$\r,$\t]) of + [] -> + lists:reverse(Terms); + _ -> + {error,{bad_term,L}} + end. diff --git a/lib/common_test/src/ct_config_xml.erl b/lib/common_test/src/ct_config_xml.erl new file mode 100644 index 0000000000..8a6e75e635 --- /dev/null +++ b/lib/common_test/src/ct_config_xml.erl @@ -0,0 +1,118 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%%---------------------------------------------------------------------- +%% File : ct_config_xml.erl +%% Description : CT callback module for reading configs from XML files +%% +%% Created : 16 February 2010 +%%---------------------------------------------------------------------- +-module(ct_config_xml). +-export([read_config/1, check_parameter/1]). + +% read config file +read_config(ConfigFile) -> + case catch do_read_xml_config(ConfigFile) of + {ok, Config}-> + {ok, Config}; + {error, Error, ErroneousString}-> + {error, Error, ErroneousString} + end. + +% check file exists +check_parameter(File)-> + case filelib:is_file(File) of + true-> + {ok, {file, File}}; + false-> + {error, {nofile, File}} + end. + +% actual reading of the config +do_read_xml_config(ConfigFile)-> + case catch xmerl_sax_parser:file(ConfigFile, + [{event_fun, fun event/3}, + {event_state, []}]) of + {ok, EntityList, _}-> + {ok, lists:reverse(transform_entity_list(EntityList))}; + Oops-> + {error, parsing_failed, Oops} + end. + +% event callback for xmerl_sax_parser +event(Event, _LineNo, State) -> + tag(Event, State). + +% document start +tag(startDocument, State) -> + State; + +% start of the config +tag({startElement, _Uri, "config", _QName, _Attributes}, []) -> + [{"config", []}]; + +% start tag +tag({startElement, _Uri, Name, _QName, _Attributes}, Tags) -> + [{Name, []}|Tags]; + +% value +tag({characters, String}, [{Tag, _Value}|Tags]) -> + [{Tag, String}|Tags]; + +% end tag +tag({endElement, _Uri, _Name, _QName}, + [Entity, {PrevEntityTag, PrevEntityValue}|Tags]) -> + NewHead = {PrevEntityTag, [Entity|PrevEntityValue]}, + [NewHead|Tags]; + +% end of the config +tag({endElement, _Uri, "config", _QName}, [{"config", Config}]) -> + Config; + +% end of document, return result +tag(endDocument, {_Tags, Result}) -> + Result; + +% default +tag(_El, State) -> + State. + +% transform of the ugly deeply nested entity list to the key-value "tree" +transform_entity_list(EntityList)-> + lists:map(fun transform_entity/1, EntityList). + +% transform entity from {list(), list()} to {atom(), term()} +transform_entity({Tag, [Value|Rest]}) when + is_tuple(Value)-> + {list_to_atom(Tag), transform_entity_list(lists:reverse([Value|Rest]))}; +transform_entity({Tag, String})-> + case list_to_term(String) of + {ok, Value}-> + {list_to_atom(Tag), Value}; + Error-> + throw(Error) + end. + +% transform a string with Erlang terms +list_to_term(String) -> + {ok, T, _} = erl_scan:string(String++"."), + case catch erl_parse:parse_term(T) of + {ok, Term} -> + {ok, Term}; + Error -> + {error, Error, String} + end. diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index ed8b564921..f2ca023cff 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -27,8 +27,12 @@ -export([init_tc/3, end_tc/3, get_suite/2, report/2, warn/1]). -export([error_notification/4]). +-export([overview_html_header/1]). + -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). +-export([make_all_conf/3, make_conf/5]). + -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -101,7 +105,8 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> [{saved_config,{LastFunc,SavedConfig}} | lists:keydelete(saved_config,1,Config0)]; {{LastSuite,InitOrEnd},SavedConfig} when InitOrEnd == init_per_suite ; - InitOrEnd == end_per_suite -> % last suite + InitOrEnd == end_per_suite -> + %% last suite [{saved_config,{LastSuite,SavedConfig}} | lists:keydelete(saved_config,1,Config0)]; undefined -> @@ -113,9 +118,9 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> ok; true -> %% delete all default values used in previous suite - ct_util:delete_default_config(suite), + ct_config:delete_default_config(suite), %% release all name -> key bindings (once per suite) - ct_util:release_allocated() + ct_config:release_allocated() end, TestCaseInfo = case catch apply(Mod,Func,[]) of @@ -125,7 +130,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> %% clear all config data default values set by previous %% testcase info function (these should only survive the %% testcase, not the whole suite) - ct_util:delete_default_config(testcase), + ct_config:delete_default_config(testcase), case add_defaults(Mod,Func,TestCaseInfo,DoInit) of Error = {suite0_failed,_} -> ct_logs:init_tc(), @@ -161,6 +166,7 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> _ -> MergeResult end, + %% timetrap must be handled before require MergedInfo1 = timetrap_first(MergedInfo, [], []), %% tell logger to use specified style sheet @@ -244,8 +250,8 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> _ -> {suite0_failed,bad_return_value} end. - -add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_) -> + +add_defaults1(_Mod,init_per_suite,[],SuiteInfo,_DoInit) -> SuiteInfo; add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) -> @@ -253,15 +259,27 @@ add_defaults1(Mod,Func,FuncInfo,SuiteInfo,DoInit) -> %% can result in weird behaviour (suite values get overwritten) SuiteReqs = [SDDef || SDDef <- SuiteInfo, - require == element(1,SDDef)], - case [element(2,Clash) || Clash <- SuiteReqs, - true == lists:keymember(element(2,Clash),2,FuncInfo)] of + ((require == element(1,SDDef)) or + (default_config == element(1,SDDef)))], + FuncReqs = + [FIDef || FIDef <- FuncInfo, + require == element(1,FIDef)], + case [element(2,Clash) || Clash <- SuiteReqs, + require == element(1, Clash), + true == lists:keymember(element(2,Clash),2, + FuncReqs)] of [] -> add_defaults2(Mod,Func,FuncInfo,SuiteInfo,SuiteReqs,DoInit); Clashes -> {error,{config_name_already_in_use,Clashes}} end. +add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,false) -> + %% not common practise to use a test case info function for + %% init_per_suite (usually handled by suite/0), but let's support + %% it just in case... + add_defaults2(Mod,init_per_suite,IPSInfo,SuiteInfo,SuiteReqs,true); + add_defaults2(_Mod,_Func,FuncInfo,SuiteInfo,_,false) -> %% include require elements from test case info, but not from suite/0 %% (since we've already required those vars) @@ -381,10 +399,10 @@ try_set_default(Name,Key,Info,Where) -> {_,[]} -> no_default; {'_UNDEF',_} -> - [ct_util:set_default_config([CfgVal],Where) || CfgVal <- CfgElems], + [ct_config:set_default_config([CfgVal],Where) || CfgVal <- CfgElems], ok; _ -> - [ct_util:set_default_config(Name,[CfgVal],Where) || CfgVal <- CfgElems], + [ct_config:set_default_config(Name,[CfgVal],Where) || CfgVal <- CfgElems], ok end. @@ -631,12 +649,12 @@ group_or_func(Func, _Config) -> %%% and every test case. If the former, all test cases in the suite %%% should be returned. -get_suite(Mod, all) -> +get_suite(Mod, all) -> case catch apply(Mod, groups, []) of {'EXIT',_} -> get_all(Mod, []); GroupDefs when is_list(GroupDefs) -> - case catch check_groups(Mod, GroupDefs) of + case catch find_groups(Mod, all, all, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly @@ -651,96 +669,193 @@ get_suite(Mod, all) -> %%!============================================================ %%! Note: The handling of sequences in get_suite/2 and get_all/2 -%%! is deprecated and should be removed after OTP R13! +%%! is deprecated and should be removed at some point... %%!============================================================ -get_suite(Mod, Name) -> - %% Name may be name of a group or a test case. If it's a group, - %% it should be expanded to list of cases (in a conf term) +%% group +get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> + Name = proplists:get_value(name, Props), case catch apply(Mod, groups, []) of {'EXIT',_} -> - get_seq(Mod, Name); + [Group]; GroupDefs when is_list(GroupDefs) -> - case catch check_groups(Mod, GroupDefs) of + case catch find_groups(Mod, Name, TCs, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; + [] -> + {error,{invalid_group_spec,Name}}; ConfTests -> - FindConf = fun({conf,Props,_,_,_}) -> - case proplists:get_value(name, Props) of - Name -> true; - _ -> false - end - end, - case lists:filter(FindConf, ConfTests) of - [] -> % must be a test case - get_seq(Mod, Name); - [ConfTest|_] -> - ConfTest + case lists:member(skipped, Props) of + true -> + %% a *subgroup* specified *only* as skipped (and not + %% as an explicit test) should not be returned, or + %% init/end functions for top groups will be executed + case catch proplists:get_value(name, element(2, hd(ConfTests))) of + Name -> % top group + ConfTests; + _ -> + [] + end; + false -> + ConfTests end end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] - end. + end; -check_groups(_Mod, []) -> - []; -check_groups(Mod, Defs) -> - check_groups(Mod, Defs, Defs, []). +%% testcase +get_suite(Mod, Name) -> + get_seq(Mod, Name). -check_groups(Mod, [TC | Gs], Defs, Levels) when is_atom(TC), length(Levels)>0 -> - [TC | check_groups(Mod, Gs, Defs, Levels)]; +%%%----------------------------------------------------------------- -check_groups(Mod, [{group,SubName} | Gs], Defs, Levels) when is_atom(SubName) -> - case lists:member(SubName, Levels) of - true -> - E = "Cyclic reference to group "++atom_to_list(SubName)++ - " in "++atom_to_list(Mod)++":groups/0", - throw({error,list_to_atom(E)}); - false -> - case find_group(Mod, SubName, Defs) of - {error,_} = Error -> - throw(Error); - G -> - [check_groups(Mod, [G], Defs, Levels) | - check_groups(Mod, Gs, Defs, Levels)] - end +find_groups(Mod, Name, TCs, GroupDefs) -> + Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), + Trimmed = trim(Found), + delete_subs(Trimmed, Trimmed). + +find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) -> + cyclic_test(Mod, Name, Known), + [make_conf(Mod, Name, Props, + find(Mod, all, all, Tests, [Name | Known], Defs, true)) | + find(Mod, all, all, Gs, [], Defs, true)]; + +find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false) + when is_atom(Name), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name, Known), + case TCs of + all -> + [make_conf(Mod, Name, Props, + find(Mod, Name, TCs, Tests, [Name | Known], Defs, true))]; + _ -> + Tests1 = [TC || TC <- TCs, + lists:member(TC, Tests) == true], + [make_conf(Mod, Name, Props, Tests1)] end; -check_groups(Mod, [{Name,Tests} | Gs], Defs, Levels) when is_atom(Name), - is_list(Tests) -> - check_groups(Mod, [{Name,[],Tests} | Gs], Defs, Levels); - -check_groups(Mod, [{Name,Props,Tests} | Gs], Defs, Levels) when is_atom(Name), - is_list(Props), - is_list(Tests) -> - {TestSpec,Levels1} = - case Levels of - [] -> - {check_groups(Mod, Tests, Defs, [Name]),[]}; - _ -> - {check_groups(Mod, Tests, Defs, [Name|Levels]),Levels} - end, - [make_conf(Mod, Name, Props, TestSpec) | - check_groups(Mod, Gs, Defs, Levels1)]; +find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false) + when is_atom(Name1), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name1, Known), + [make_conf(Mod, Name1, Props, + find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | + find(Mod, Name, TCs, Gs, [], Defs, false)]; + +find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true) + when is_atom(Name) -> + E = "Duplicate groups named "++atom_to_list(Name)++" in "++ + atom_to_list(Mod)++":groups/0", + throw({error,list_to_atom(E)}); + +find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true) + when is_atom(Name1), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name1, Known), + [make_conf(Mod, Name1, Props, + find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) | + find(Mod, Name, all, Gs, [], Defs, true)]; + +find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) when is_atom(Name1) -> + find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found); -check_groups(Mod, [BadTerm | _Gs], _Defs, Levels) -> - Where = if length(Levels) == 0 -> +find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found) + when is_atom(Name1), is_list(Tests) -> + find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found); + +find(Mod, Name, TCs, [TC | Gs], Known, Defs, false) when is_atom(TC) -> + find(Mod, Name, TCs, Gs, Known, Defs, false); + +find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) -> + [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)]; + +find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) -> + Where = if length(Known) == 0 -> atom_to_list(Mod)++":groups/0"; true -> - "group "++atom_to_list(lists:last(Levels))++ + "group "++atom_to_list(lists:last(Known))++ " in "++atom_to_list(Mod)++":groups/0" end, Term = io_lib:format("~p", [BadTerm]), E = "Bad term "++lists:flatten(Term)++" in "++Where, throw({error,list_to_atom(E)}); -check_groups(_Mod, [], _Defs, _) -> +find(_Mod, _Name, _TCs, [], _Known, _Defs, false) -> + ['$NOMATCH']; + +find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) -> []. -find_group(Mod, Name, Defs) -> +delete_subs([Conf | Confs], All) -> + All1 = delete_conf(Conf, All), + case is_sub(Conf, All1) of + true -> + delete_subs(Confs, All1); + false -> + delete_subs(Confs, All) + end; + +delete_subs([], All) -> + All. + +delete_conf({conf,Props,_,_,_}, Confs) -> + Name = proplists:get_value(name, Props), + [Conf || Conf = {conf,Props0,_,_,_} <- Confs, + Name =/= proplists:get_value(name, Props0)]. + +is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> + Name = proplists:get_value(name, Props), + case lists:any(fun({conf,Props0,_,_,_}) -> + case proplists:get_value(name, Props0) of + N when N == Name -> + true; + _ -> + false + end; + (_) -> + false + end, Tests) of + true -> + true; + false -> + is_sub(Conf, Tests) or is_sub(Conf, Confs) + end; + +is_sub(Conf, [_TC | Tests]) -> + is_sub(Conf, Tests); + +is_sub(_Conf, []) -> + false. + +trim(['$NOMATCH' | Tests]) -> + trim(Tests); + +trim([{conf,Props,Init,Tests,End} | Confs]) -> + case trim(Tests) of + [] -> + trim(Confs); + Trimmed -> + [{conf,Props,Init,Trimmed,End} | trim(Confs)] + end; + +trim([TC | Tests]) -> + [TC | trim(Tests)]; + +trim([]) -> + []. + +cyclic_test(Mod, Name, Names) -> + case lists:member(Name, Names) of + true -> + E = "Cyclic reference to group "++atom_to_list(Name)++ + " in "++atom_to_list(Mod)++":groups/0", + throw({error,list_to_atom(E)}); + false -> + ok + end. + +expand(Mod, Name, Defs) -> case lists:keysearch(Name, 1, Defs) of {value,Def} -> Def; @@ -750,7 +865,48 @@ find_group(Mod, Name, Defs) -> throw({error,list_to_atom(E)}) end. +make_all_conf(Dir, Mod, _Props) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_all_conf(Mod). + +make_all_conf(Mod) -> + case catch apply(Mod, groups, []) of + {'EXIT',_} -> + {error,{invalid_group_definition,Mod}}; + GroupDefs when is_list(GroupDefs) -> + case catch find_groups(Mod, all, all, GroupDefs) of + {error,_} = Error -> + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Error properly + [{?MODULE,error_in_suite,[[Error]]}]; + [] -> + {error,{invalid_group_spec,Mod}}; + ConfTests -> + [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests] + end + end. + +make_conf(Dir, Mod, Name, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_conf(Mod, Name, Props, TestSpec). + make_conf(Mod, Name, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_file(Mod); + _ -> + ok + end, {InitConf,EndConf} = case erlang:function_exported(Mod,init_per_group,2) of true -> @@ -761,6 +917,7 @@ make_conf(Mod, Name, Props, TestSpec) -> end, {conf,[{name,Name}|Props],InitConf,TestSpec,EndConf}. +%%%----------------------------------------------------------------- get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of @@ -1076,4 +1233,31 @@ add_data_dir(File,Config) when is_list(File) -> File end. +%%%----------------------------------------------------------------- +%%% @spec overview_html_header(TestName) -> Header +overview_html_header(TestName) -> + TestName1 = lists:flatten(io_lib:format("~p", [TestName])), + Label = case application:get_env(common_test, test_label) of + {ok,Lbl} when Lbl =/= undefined -> + "<H1><FONT color=\"green\">" ++ Lbl ++ "</FONT></H1>\n"; + _ -> + "" + end, + Bgr = case ct_logs:basic_html() of + true -> + ""; + false -> + CTPath = code:lib_dir(common_test), + TileFile = filename:join(filename:join(CTPath,"priv"),"tile1.jpg"), + " background=\"" ++ TileFile ++ "\"" + end, + + ["<html>\n", + "<head><title>Test ", TestName1, " results</title>\n", + "<meta http-equiv=\"cache-control\" content=\"no-cache\">\n", + "</head>\n", + "<body", Bgr, " bgcolor=\"white\" text=\"black\" ", + "link=\"blue\" vlink=\"purple\" alink=\"red\">\n", + Label, + "<H2>Results from test ", TestName1, "</H2>\n"]. diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index a31e57c7ea..5aab4dd2dd 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -76,7 +76,7 @@ start(Name,Address,InitData,CallbackMod) -> MRef = erlang:monitor(process,Pid), receive {connected,Pid} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), ct_util:register_connection(Name,Address,CallbackMod,Pid), {ok,Pid}; {Error,Pid} -> @@ -182,7 +182,7 @@ call(Pid,Msg) -> Pid ! {Msg,{self(),Ref}}, receive {Ref, Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), case Result of {retry,_Data} -> call(Pid,Result); diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index bd1a89ae1f..3684ff462f 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -36,7 +36,8 @@ -export([make_all_suites_index/1,make_all_runs_index/1]). %% Logging stuff directly from testcase --export([tc_log/3,tc_print/3,tc_pal/3]). +-export([tc_log/3,tc_print/3,tc_pal/3, + basic_html/0]). %% Simulate logger process for use without ct environment running -export([simulate/0]). @@ -57,7 +58,7 @@ -define(table_color2,"#E4F0FE"). -define(table_color3,"#F0F8FF"). --define(testname_width, 70). +-define(testname_width, 60). -define(abs(Name), filename:absname(Name)). @@ -80,7 +81,7 @@ init(Mode) -> MRef = erlang:monitor(process,Pid), receive {started,Pid,Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), Result; {'DOWN',MRef,process,_,Reason} -> exit({could_not_start_process,?MODULE,Reason}) @@ -163,7 +164,7 @@ call(Msg) -> ?MODULE ! {Msg,{self(),Ref}}, receive {Ref, Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), Result; {'DOWN',MRef,process,_,Reason} -> {error,{process_down,?MODULE,Reason}} @@ -383,11 +384,14 @@ maybe_log_timestamp() -> [{"<i>~s</i>",[log_timestamp({MS,S,US})]}]}) end. -log_timestamp(Now) -> - put(log_timestamp,Now), - {_,{H,M,S}} = calendar:now_to_local_time(Now), - lists:flatten(io_lib:format("~2.2.0w:~2.2.0w:~2.2.0w", - [H,M,S])). +log_timestamp({MS,S,US}) -> + put(log_timestamp, {MS,S,US}), + {{Year,Month,Day}, {Hour,Min,Sec}} = + calendar:now_to_local_time({MS,S,US}), + MilliSec = trunc(US/1000), + lists:flatten(io_lib:format("~4.10.0B-~2.10.0B-~2.10.0B " + "~2.10.0B:~2.10.0B:~2.10.0B.~3.10.0B", + [Year,Month,Day,Hour,Min,Sec,MilliSec])). %%%----------------------------------------------------------------- %%% The logger server @@ -505,7 +509,7 @@ logger_loop(State) -> logger_loop(State); {set_stylesheet,TC,SSFile} -> Fd = State#logger_state.ct_log_fd, - io:format(Fd, "~p uses external style sheet: ~s~n", [TC,SSFile]), + io:format(Fd, "~p loading external style sheet: ~s~n", [TC,SSFile]), logger_loop(State#logger_state{stylesheet=SSFile}); {clear_stylesheet,_} when State#logger_state.stylesheet == undefined -> logger_loop(State); @@ -716,7 +720,7 @@ make_last_run_index1(StartTime,IndexName) -> [Log]; Logs -> case read_totals_file(?totals_name) of - {_Node,Logs0,_Totals} -> + {_Node,_Lbl,Logs0,_Totals} -> insert_dirs(Logs,Logs0); _ -> %% someone deleted the totals file!? @@ -728,10 +732,15 @@ make_last_run_index1(StartTime,IndexName) -> {ok,Bin} -> binary_to_term(Bin); _ -> [] end, - {ok,Index0,Totals} = make_last_run_index(Logs1, index_header(StartTime), + Label = case application:get_env(common_test, test_label) of + {ok,Lbl} -> Lbl; + _ -> undefined + end, + {ok,Index0,Totals} = make_last_run_index(Logs1, + index_header(Label,StartTime), 0, 0, 0, 0, 0, Missing), %% write current Totals to file, later to be used in all_runs log - write_totals_file(?totals_name,Logs1,Totals), + write_totals_file(?totals_name,Label,Logs1,Totals), Index = [Index0|index_footer()], case force_write_file(IndexName, Index) of ok -> @@ -761,7 +770,7 @@ make_last_run_index([Name|Rest], Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, Missing); LastLogDir -> SuiteName = filename:rootname(filename:basename(Name)), - case make_one_index_entry(SuiteName, LastLogDir, false, Missing) of + case make_one_index_entry(SuiteName, LastLogDir, "-", false, Missing) of {Result1,Succ,Fail,USkip,ASkip,NotBuilt} -> %% for backwards compatibility AutoSkip1 = case catch AutoSkip+ASkip of @@ -780,18 +789,18 @@ make_last_run_index([], Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuil {ok, [Result|total_row(TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, false)], {TotSucc,TotFail,UserSkip,AutoSkip,TotNotBuilt}}. -make_one_index_entry(SuiteName, LogDir, All, Missing) -> +make_one_index_entry(SuiteName, LogDir, Label, All, Missing) -> case count_cases(LogDir) of {Succ,Fail,UserSkip,AutoSkip} -> NotBuilt = not_built(SuiteName, LogDir, All, Missing), - NewResult = make_one_index_entry1(SuiteName, LogDir, Succ, Fail, + NewResult = make_one_index_entry1(SuiteName, LogDir, Label, Succ, Fail, UserSkip, AutoSkip, NotBuilt, All), {NewResult,Succ,Fail,UserSkip,AutoSkip,NotBuilt}; error -> error end. -make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, +make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> LogFile = filename:join(Link, ?suitelog_name ++ ".html"), CrashDumpName = SuiteName ++ "_erl_crash.dump", @@ -803,7 +812,7 @@ make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, false -> "" end, - {Timestamp,Node,AllInfo} = + {Lbl,Timestamp,Node,AllInfo} = case All of {true,OldRuns} -> [_Prefix,NodeOrDate|_] = string:tokens(Link,"."), @@ -811,20 +820,21 @@ make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, 0 -> "-"; _ -> NodeOrDate end, - N = ["<TD ALIGN=right>",Node1,"</TD>\n"], + N = ["<TD ALIGN=right><FONT SIZE=-1>",Node1,"</FONT></TD>\n"], CtRunDir = filename:dirname(filename:dirname(Link)), - T = ["<TD>",timestamp(CtRunDir),"</TD>\n"], + L = ["<TD ALIGN=center><FONT SIZE=-1><B>",Label,"</FONT></B></TD>\n"], + T = ["<TD><FONT SIZE=-1>",timestamp(CtRunDir),"</FONT></TD>\n"], CtLogFile = filename:join(CtRunDir,?ct_log_name), OldRunsLink = case OldRuns of [] -> "none"; _ -> "<A HREF=\""++?all_runs_name++"\">Old Runs</A>" end, - A=["<TD><A HREF=\"",CtLogFile,"\">CT Log</A></TD>\n", - "<TD>",OldRunsLink,"</TD>\n"], - {T,N,A}; + A=["<TD><FONT SIZE=-1><A HREF=\"",CtLogFile,"\">CT Log</A></FONT></TD>\n", + "<TD><FONT SIZE=-1>",OldRunsLink,"</FONT></TD>\n"], + {L,T,N,A}; false -> - {"","",""} + {"","","",""} end, NotBuiltStr = if NotBuilt == 0 -> @@ -851,7 +861,8 @@ make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, {UserSkip+AutoSkip,integer_to_list(UserSkip),ASStr} end, ["<TR valign=top>\n", - "<TD><A HREF=\"",LogFile,"\">",SuiteName,"</A>",CrashDumpLink,"</TD>\n", + "<TD><FONT SIZE=-1><A HREF=\"",LogFile,"\">",SuiteName,"</A>",CrashDumpLink,"</FONT></TD>\n", + Lbl, Timestamp, "<TD ALIGN=right>",integer_to_list(Success),"</TD>\n", "<TD ALIGN=right>",FailStr,"</TD>\n", @@ -862,12 +873,14 @@ make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, AllInfo, "</TR>\n"]. total_row(Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> - {TimestampCell,AllInfo} = + {Label,TimestampCell,AllInfo} = case All of - true -> - {"<TD> </TD>\n","<TD> </TD>\n<TD> </TD>\n"}; + true -> + {"<TD> </TD>\n", + "<TD> </TD>\n", + "<TD> </TD>\n<TD> </TD>\n"}; false -> - {"",""} + {"","",""} end, {AllSkip,UserSkipStr,AutoSkipStr} = @@ -877,6 +890,7 @@ total_row(Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> end, ["<TR valign=top>\n", "<TD><B>Total</B></TD>", + Label, TimestampCell, "<TD ALIGN=right><B>",integer_to_list(Success),"<B></TD>\n", "<TD ALIGN=right><B>",integer_to_list(Fail),"<B></TD>\n", @@ -937,13 +951,21 @@ term_to_text(Term) -> %%% Headers and footers. -index_header(StartTime) -> - [header("Test Results " ++ format_time(StartTime)) | +index_header(Label, StartTime) -> + Head = + case Label of + undefined -> + header("Test Results", format_time(StartTime)); + _ -> + header("Test Results for \"" ++ Label ++ "\"", + format_time(StartTime)) + end, + [Head | ["<CENTER>\n", "<P><A HREF=\"",?ct_log_name,"\">Common Test Framework Log</A></P>", "<TABLE border=\"3\" cellpadding=\"5\" " "BGCOLOR=\"",?table_color3,"\">\n" - "<th><B>Name</B></th>\n", + "<th><B>Test Name</B></th>\n", "<th><font color=\"",?table_color3,"\">_</font>Ok" "<font color=\"",?table_color3,"\">_</font></th>\n" "<th>Failed</th>\n", @@ -952,13 +974,17 @@ index_header(StartTime) -> "\n"]]. all_suites_index_header() -> + {ok,Cwd} = file:get_cwd(), + LogDir = filename:basename(Cwd), + AllRuns = "All test runs in \"" ++ LogDir ++ "\"", [header("Test Results") | ["<CENTER>\n", - "<A HREF=\"",?all_runs_name,"\">All Test Runs in this directory</A>\n", + "<A HREF=\"",?all_runs_name,"\">",AllRuns,"</A>\n", "<br><br>\n", "<TABLE border=\"3\" cellpadding=\"5\" " "BGCOLOR=\"",?table_color2,"\">\n" - "<th>Name</th>\n", + "<th>Test Name</th>\n", + "<th>Label</th>\n", "<th>Test Run Started</th>\n", "<th><font color=\"",?table_color2,"\">_</font>Ok" "<font color=\"",?table_color2,"\">_</font></th>\n" @@ -971,13 +997,17 @@ all_suites_index_header() -> "\n"]]. all_runs_header() -> - [header("All test runs in current directory") | + {ok,Cwd} = file:get_cwd(), + LogDir = filename:basename(Cwd), + Title = "All test runs in \"" ++ LogDir ++ "\"", + [header(Title) | ["<CENTER><TABLE border=\"3\" cellpadding=\"5\" " "BGCOLOR=\"",?table_color1,"\">\n" "<th><B>History</B></th>\n" "<th><B>Node</B></th>\n" + "<th><B>Label</B></th>\n" "<th>Tests</th>\n" - "<th><B>Names</B></th>\n" + "<th><B>Test Names</B></th>\n" "<th>Total</th>\n" "<th><font color=\"",?table_color1,"\">_</font>Ok" "<font color=\"",?table_color1,"\">_</font></th>\n" @@ -987,12 +1017,23 @@ all_runs_header() -> "\n"]]. header(Title) -> + header1(Title, ""). +header(Title, SubTitle) -> + header1(Title, SubTitle). + +header1(Title, SubTitle) -> + SubTitleHTML = if SubTitle =/= "" -> + ["<CENTER>\n", + "<H2>" ++ SubTitle ++ "</H2>\n", + "</CENTER>\n<BR>\n"]; + true -> "<BR>\n" + end, ["<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\">\n" "<!-- autogenerated by '"++atom_to_list(?MODULE)++"'. -->\n" "<HTML>\n", "<HEAD>\n", - "<TITLE>" ++ Title ++ "</TITLE>\n", + "<TITLE>" ++ Title ++ " " ++ SubTitle ++ "</TITLE>\n", "<META HTTP-EQUIV=\"CACHE-CONTROL\" CONTENT=\"NO-CACHE\">\n", "</HEAD>\n", @@ -1004,6 +1045,7 @@ header(Title) -> "<CENTER>\n", "<H1>" ++ Title ++ "</H1>\n", "</CENTER>\n", + SubTitleHTML, "<!-- ---- CONTENT ---- -->\n"]. @@ -1013,19 +1055,28 @@ index_footer() -> footer() -> ["<P><CENTER>\n" - "<HR>\n" - "<P><FONT SIZE=-1>\n" - "Copyright © ", year(), - " <A HREF=\"http://erlang.ericsson.se\">Open Telecom Platform</A><BR>\n" - "Updated: <!date>", current_time(), "<!/date><BR>\n" - "</FONT>\n" - "</CENTER>\n" - "</body>\n"]. + "<BR><BR>\n" + "<HR>\n" + "<P><FONT SIZE=-1>\n" + "Copyright © ", year(), + " <A HREF=\"http://erlang.ericsson.se\">Open Telecom Platform</A><BR>\n" + "Updated: <!date>", current_time(), "<!/date><BR>\n" + "</FONT>\n" + "</CENTER>\n" + "</body>\n"]. body_tag() -> - "<body bgcolor=\"#FFFFFF\" text=\"#000000\" link=\"#0000FF\"" - "vlink=\"#800080\" alink=\"#FF0000\">\n". + case basic_html() of + true -> + "<body bgcolor=\"#FFFFFF\" text=\"#000000\" link=\"#0000FF\" " + "vlink=\"#800080\" alink=\"#FF0000\">\n"; + false -> + CTPath = code:lib_dir(common_test), + TileFile = filename:join(filename:join(CTPath,"priv"),"tile1.jpg"), + "<body background=\"" ++ TileFile ++ "\" bgcolor=\"#FFFFFF\" text=\"#000000\" link=\"#0000FF\" " + "vlink=\"#800080\" alink=\"#FF0000\">\n" + end. current_time() -> format_time(calendar:local_time()). @@ -1217,7 +1268,7 @@ runentry(Dir, BasicHtml) -> TotalsFile = filename:join(Dir,?totals_name), TotalsStr = case read_totals_file(TotalsFile) of - {Node,Logs,{TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}} -> + {Node,Label,Logs,{TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}} -> TotFailStr = if TotFail > 0 -> ["<FONT color=\"red\">", @@ -1263,6 +1314,7 @@ runentry(Dir, BasicHtml) -> end, Total = TotSucc+TotFail+AllSkip, A = ["<TD ALIGN=center><FONT SIZE=-1>",Node,"</FONT></TD>\n", + "<TD ALIGN=center><FONT SIZE=-1><B>",Label,"</B></FONT></TD>\n", "<TD ALIGN=right>",NoOfTests,"</TD>\n"], B = if BasicHtml -> ["<TD ALIGN=center><FONT SIZE=-1>",TestNamesTrunc,"</FONT></TD>\n"]; @@ -1283,17 +1335,19 @@ runentry(Dir, BasicHtml) -> end, Index = filename:join(Dir,?index_name), ["<TR>\n" - "<TD><A HREF=\"",Index,"\">",timestamp(Dir),"</A>",TotalsStr,"</TD>\n" + "<TD><FONT SIZE=-1><A HREF=\"",Index,"\">",timestamp(Dir),"</A>",TotalsStr,"</FONT></TD>\n" "</TR>\n"]. -write_totals_file(Name,Logs,Totals) -> +write_totals_file(Name,Label,Logs,Totals) -> AbsName = ?abs(Name), notify_and_lock_file(AbsName), force_write_file(AbsName, term_to_binary({atom_to_list(node()), - Logs,Totals})), + Label,Logs,Totals})), notify_and_unlock_file(AbsName). +%% this function needs to convert from old formats to new so that old +%% test results (prev ct versions) can be listed together with new read_totals_file(Name) -> AbsName = ?abs(Name), notify_and_lock_file(AbsName), @@ -1303,12 +1357,23 @@ read_totals_file(Name) -> case catch binary_to_term(Bin) of {'EXIT',_Reason} -> % corrupt file {"-",[],undefined}; - R = {Node,Ls,Tot} -> + {Node,Label,Ls,Tot} -> % all info available + Label1 = case Label of + undefined -> "-"; + _ -> Label + end, case Tot of - {_,_,_,_,_} -> % latest format - R; + {_Ok,_Fail,_USkip,_ASkip,_NoBuild} -> % latest format + {Node,Label1,Ls,Tot}; {TotSucc,TotFail,AllSkip,NotBuilt} -> - {Node,Ls,{TotSucc,TotFail,AllSkip,undefined,NotBuilt}} + {Node,Label1,Ls,{TotSucc,TotFail,AllSkip,undefined,NotBuilt}} + end; + {Node,Ls,Tot} -> % no label found + case Tot of + {_Ok,_Fail,_USkip,_ASkip,_NoBuild} -> % latest format + {Node,"-",Ls,Tot}; + {TotSucc,TotFail,AllSkip,NotBuilt} -> + {Node,"-",Ls,{TotSucc,TotFail,AllSkip,undefined,NotBuilt}} end; %% for backwards compatibility {Ls,Tot} -> {"-",Ls,Tot}; @@ -1411,7 +1476,7 @@ make_all_suites_index1(When,AllSuitesLogDirs) -> make_all_suites_index2(IndexName,AllSuitesLogDirs) -> {ok,Index0,_Totals} = make_all_suites_index3(AllSuitesLogDirs, all_suites_index_header(), - 0, 0, 0, 0, 0), + 0, 0, 0, 0, 0, []), Index = [Index0|index_footer()], case force_write_file(IndexName, Index) of ok -> @@ -1421,14 +1486,25 @@ make_all_suites_index2(IndexName,AllSuitesLogDirs) -> end. make_all_suites_index3([{SuiteName,[LastLogDir|OldDirs]}|Rest], - Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt) -> + Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, + Labels) -> [EntryDir|_] = filename:split(LastLogDir), Missing = case file:read_file(filename:join(EntryDir,?missing_suites_info)) of {ok,Bin} -> binary_to_term(Bin); _ -> [] end, - case make_one_index_entry(SuiteName, LastLogDir, {true,OldDirs}, Missing) of + {Label,Labels1} = + case proplists:get_value(EntryDir, Labels) of + undefined -> + case read_totals_file(filename:join(EntryDir,?totals_name)) of + {_,Lbl,_,_} -> {Lbl,[{EntryDir,Lbl}|Labels]}; + _ -> {"-",[{EntryDir,"-"}|Labels]} + end; + Lbl -> + {Lbl,Labels} + end, + case make_one_index_entry(SuiteName, LastLogDir, Label, {true,OldDirs}, Missing) of {Result1,Succ,Fail,USkip,ASkip,NotBuilt} -> %% for backwards compatibility AutoSkip1 = case catch AutoSkip+ASkip of @@ -1437,13 +1513,13 @@ make_all_suites_index3([{SuiteName,[LastLogDir|OldDirs]}|Rest], end, make_all_suites_index3(Rest, [Result|Result1], TotSucc+Succ, TotFail+Fail, UserSkip+USkip, AutoSkip1, - TotNotBuilt+NotBuilt); + TotNotBuilt+NotBuilt,Labels1); error -> make_all_suites_index3(Rest, Result, TotSucc, TotFail, - UserSkip, AutoSkip, TotNotBuilt) + UserSkip, AutoSkip, TotNotBuilt,Labels1) end; make_all_suites_index3([], Result, TotSucc, TotFail, UserSkip, AutoSkip, - TotNotBuilt) -> + TotNotBuilt,_) -> {ok, [Result|total_row(TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt,true)], {TotSucc,TotFail,UserSkip,AutoSkip,TotNotBuilt}}. diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index 7eb2c3cfef..42e4cf08f4 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2006-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -28,7 +28,7 @@ -export([abort/0,abort/1,progress/0]). --export([init_master/6, init_node_ctrl/3]). +-export([init_master/7, init_node_ctrl/3]). -export([status/2]). @@ -49,7 +49,8 @@ %%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} | %%% {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} | %%% {logdir,LogDir} | {event_handler,EventHandlers} | -%%% {silent_connections,Conns} | {cover,CoverSpecFile} +%%% {silent_connections,Conns} | {cover,CoverSpecFile} | +%%% {userconfig, UserCfgFiles} %%% CfgFiles = string() | [string()] %%% TestDirs = string() | [string()] %%% Suites = atom() | [atom()] @@ -98,11 +99,14 @@ run([TS|TestSpecs],AllowUserTerms,InclNodes,ExclNodes) when is_list(TS), {error,Reason} -> {error,Reason}; TSRec=#testspec{logdir=AllLogDirs, - config=AllCfgFiles, + config=StdCfgFiles, + userconfig=UserCfgFiles, + init=AllInitOpts, event_handler=AllEvHs} -> + AllCfgFiles = {StdCfgFiles, UserCfgFiles}, RunSkipPerNode = ct_testspec:prepare_tests(TSRec), RunSkipPerNode2 = exclude_nodes(ExclNodes,RunSkipPerNode), - run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs,[],[],TS1) + run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs,[],[],AllInitOpts,TS1) end, [{TS,Result} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)]; run([],_,_,_) -> @@ -157,10 +161,13 @@ run_on_node([TS|TestSpecs],AllowUserTerms,Node) when is_list(TS),is_atom(Node) - {error,Reason} -> {error,Reason}; TSRec=#testspec{logdir=AllLogDirs, - config=AllCfgFiles, + config=StdCfgFiles, + init=AllInitOpts, + userconfig=UserCfgFiles, event_handler=AllEvHs} -> + AllCfgFiles = {StdCfgFiles, UserCfgFiles}, {Run,Skip} = ct_testspec:prepare_tests(TSRec,Node), - run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs,[],[],TS1) + run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs,[],[],AllInitOpts,TS1) end, [{TS,Result} | run_on_node(TestSpecs,AllowUserTerms,Node)]; run_on_node([],_,_) -> @@ -180,7 +187,9 @@ run_on_node(TestSpecs,Node) -> -run_all([{Node,Run,Skip}|Rest],AllLogDirs,AllCfgFiles,AllEvHs,NodeOpts,LogDirs,Specs) -> +run_all([{Node,Run,Skip}|Rest],AllLogDirs, + {AllStdCfgFiles, AllUserCfgFiles}=AllCfgFiles, + AllEvHs,NodeOpts,LogDirs,InitOptions,Specs) -> LogDir = lists:foldl(fun({N,Dir},_Found) when N == Node -> Dir; @@ -191,29 +200,36 @@ run_all([{Node,Run,Skip}|Rest],AllLogDirs,AllCfgFiles,AllEvHs,NodeOpts,LogDirs,S (_Dir,Found) -> Found end,".",AllLogDirs), - CfgFiles = + + StdCfgFiles = lists:foldr(fun({N,F},Fs) when N == Node -> [F|Fs]; ({_N,_F},Fs) -> Fs; (F,Fs) -> [F|Fs] - end,[],AllCfgFiles), + end,[],AllStdCfgFiles), + UserCfgFiles = + lists:foldr(fun({N,F},Fs) when N == Node -> [{userconfig, F}|Fs]; + ({_N,_F},Fs) -> Fs; + (F,Fs) -> [{userconfig, F}|Fs] + end,[],AllUserCfgFiles), EvHs = lists:foldr(fun({N,H,A},Hs) when N == Node -> [{H,A}|Hs]; ({_N,_H,_A},Hs) -> Hs; ({H,A},Hs) -> [{H,A}|Hs] end,[],AllEvHs), + NO = {Node,[{prepared_tests,{Run,Skip},Specs}, {logdir,LogDir}, - {config,CfgFiles}, - {event_handler,EvHs}]}, - run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,[NO|NodeOpts],[LogDir|LogDirs],Specs); -run_all([],AllLogDirs,_,AllEvHs,NodeOpts,LogDirs,Specs) -> + {config,StdCfgFiles}, + {event_handler,EvHs}] ++ UserCfgFiles}, + run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,[NO|NodeOpts],[LogDir|LogDirs],InitOptions,Specs); +run_all([],AllLogDirs,_,AllEvHs,NodeOpts,LogDirs,InitOptions,Specs) -> Handlers = [{H,A} || {Master,H,A} <- AllEvHs, Master == master], MasterLogDir = case lists:keysearch(master,1,AllLogDirs) of {value,{_,Dir}} -> Dir; false -> "." end, log(tty,"Master Logdir","~s",[MasterLogDir]), - start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,LogDirs,Specs), + start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,LogDirs,InitOptions,Specs), ok. @@ -251,17 +267,17 @@ progress() -> %%% MASTER, runs on central controlling node. %%%----------------------------------------------------------------- start_master(NodeOptsList) -> - start_master(NodeOptsList,[],".",[],[]). + start_master(NodeOptsList,[],".",[],[],[]). -start_master(NodeOptsList,EvHandlers,MasterLogDir,LogDirs,Specs) -> +start_master(NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Specs) -> Master = spawn_link(?MODULE,init_master,[self(),NodeOptsList,EvHandlers, - MasterLogDir,LogDirs,Specs]), + MasterLogDir,LogDirs,InitOptions,Specs]), receive {Master,Result} -> Result end. %%% @hidden -init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,Specs) -> +init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Specs) -> case whereis(ct_master) of undefined -> register(ct_master,self()), @@ -314,10 +330,10 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,Specs) -> Pid when is_pid(Pid) -> ok end, - init_master1(Parent,NodeOptsList,LogDirs). + init_master1(Parent,NodeOptsList,InitOptions,LogDirs). -init_master1(Parent,NodeOptsList,LogDirs) -> - {Inaccessible,NodeOptsList1} = ping_nodes(NodeOptsList,[],[]), +init_master1(Parent,NodeOptsList,InitOptions,LogDirs) -> + {Inaccessible,NodeOptsList1,InitOptions1} = init_nodes(NodeOptsList,InitOptions), case Inaccessible of [] -> init_master2(Parent,NodeOptsList,LogDirs); @@ -331,7 +347,7 @@ init_master1(Parent,NodeOptsList,LogDirs) -> "Proceeding without: ~p",[Inaccessible]), init_master2(Parent,NodeOptsList1,LogDirs); "r\n" -> - init_master1(Parent,NodeOptsList,LogDirs); + init_master1(Parent,NodeOptsList,InitOptions1,LogDirs); _ -> log(html,"Aborting Tests","",[]), ct_master_event:stop(), @@ -542,6 +558,9 @@ get_pid(Node,NodeCtrlPids) -> undefined end. +ping_nodes(NodeOptions)-> + ping_nodes(NodeOptions, [], []). + ping_nodes([NO={Node,_Opts}|NOs],Inaccessible,NodeOpts) -> case net_adm:ping(Node) of pong -> @@ -678,13 +697,80 @@ call(Pid,Msg) -> {'DOWN', Ref, _, _, _} -> {error,master_died} end, - erlang:demonitor(Ref), + erlang:demonitor(Ref, [flush]), Return. reply(Result,To) -> To ! {self(),Result}, ok. +init_nodes(NodeOptions, InitOptions)-> + ping_nodes(NodeOptions), + start_nodes(InitOptions), + eval_on_nodes(InitOptions), + {Inaccessible, NodeOptions1}=ping_nodes(NodeOptions), + InitOptions1 = filter_accessible(InitOptions, Inaccessible), + {Inaccessible, NodeOptions1, InitOptions1}. + +% only nodes which are inaccessible now, should be initiated later +filter_accessible(InitOptions, Inaccessible)-> + [{Node,Option}||{Node,Option}<-InitOptions, lists:member(Node, Inaccessible)]. + +start_nodes(InitOptions)-> + lists:foreach(fun({NodeName, Options})-> + [NodeS,HostS]=string:tokens(atom_to_list(NodeName), "@"), + Node=list_to_atom(NodeS), + Host=list_to_atom(HostS), + HasNodeStart = lists:keymember(node_start, 1, Options), + IsAlive = lists:member(NodeName, nodes()), + case {HasNodeStart, IsAlive} of + {false, false}-> + io:format("WARNING: Node ~p is not alive but has no node_start option~n", [NodeName]); + {false, true}-> + io:format("Node ~p is alive~n", [NodeName]); + {true, false}-> + {node_start, NodeStart} = lists:keyfind(node_start, 1, Options), + {value, {callback_module, Callback}, NodeStart2}= + lists:keytake(callback_module, 1, NodeStart), + case Callback:start(Host, Node, NodeStart2) of + {ok, NodeName} -> + io:format("Node ~p started successfully with callback ~p~n", [NodeName,Callback]); + {error, Reason, _NodeName} -> + io:format("Failed to start node ~p with callback ~p! Reason: ~p~n", [NodeName, Callback, Reason]) + end; + {true, true}-> + io:format("WARNING: Node ~p is alive but has node_start option~n", [NodeName]) + end + end, + InitOptions). + +eval_on_nodes(InitOptions)-> + lists:foreach(fun({NodeName, Options})-> + HasEval = lists:keymember(eval, 1, Options), + IsAlive = lists:member(NodeName, nodes()), + case {HasEval, IsAlive} of + {false,_}-> + ok; + {true,false}-> + io:format("WARNING: Node ~p is not alive but has eval option ~n", [NodeName]); + {true,true}-> + {eval, MFAs} = lists:keyfind(eval, 1, Options), + evaluate(NodeName, MFAs) + end + end, + InitOptions). + +evaluate(Node, [{M,F,A}|MFAs])-> + case rpc:call(Node, M, F, A) of + {badrpc,Reason}-> + io:format("WARNING: Failed to call ~p:~p/~p on node ~p due to ~p~n", [M,F,length(A),Node,Reason]); + Result-> + io:format("Called ~p:~p/~p on node ~p, result: ~p~n", [M,F,length(A),Node,Result]) + end, + evaluate(Node, MFAs); +evaluate(_Node, [])-> + ok. + %cast(Msg) -> % cast(whereis(ct_master),Msg). diff --git a/lib/common_test/src/ct_master_logs.erl b/lib/common_test/src/ct_master_logs.erl index 63f60b1182..244faace06 100644 --- a/lib/common_test/src/ct_master_logs.erl +++ b/lib/common_test/src/ct_master_logs.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2006-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -44,7 +44,7 @@ start(LogDir,Nodes) -> MRef = erlang:monitor(process,Pid), receive {started,Pid,Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), {Pid,Result}; {'DOWN',MRef,process,_,Reason} -> exit({could_not_start_process,?MODULE,Reason}) @@ -435,7 +435,7 @@ call(Msg) -> ?MODULE ! {Msg,{self(),Ref}}, receive {Ref, Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), Result; {'DOWN',MRef,process,_,Reason} -> {error,{process_down,?MODULE,Reason}} diff --git a/lib/common_test/src/ct_repeat.erl b/lib/common_test/src/ct_repeat.erl index 7ac6e045d7..be3c485b75 100644 --- a/lib/common_test/src/ct_repeat.erl +++ b/lib/common_test/src/ct_repeat.erl @@ -1,26 +1,26 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% %%% @doc Common Test Framework module that handles repeated test runs %%% %%% <p>This module exports functions for repeating tests. The following -%%% script flags (or equivalent ct:run_test/1 options) are supported: +%%% start flags (or equivalent ct:run_test/1 options) are supported: %%% -until <StopTime>, StopTime = YYMoMoDDHHMMSS | HHMMSS %%% -duration <DurTime>, DurTime = HHMMSS %%% -force_stop diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 6b1063f74c..c5bfd01642 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -24,7 +24,6 @@ -module(ct_run). - %% Script interface -export([script_start/0,script_usage/0]). @@ -46,11 +45,28 @@ -define(abs(Name), filename:absname(Name)). -define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)). +-record(opts, {label, + vts, + shell, + cover, + coverspec, + step, + logdir, + config = [], + event_handlers = [], + include = [], + silent_connections, + stylesheet, + multiply_timetraps = 1, + scale_timetraps = false, + testspecs = [], + tests}). + %%%----------------------------------------------------------------- %%% @spec script_start() -> void() %%% -%%% @doc Start tests via the run_test script. -%%% +%%% @doc Start tests via the run_test program or script. +%%% %%% <p>Example:<br/><code>./run_test -config config.ctc -dir %%% $TEST_DIR</code></p> %%% @@ -59,13 +75,53 @@ %%% script_start() -> process_flag(trap_exit, true), - Args = merge_arguments(init:get_arguments()), + Init = init:get_arguments(), + CtArgs = lists:takewhile(fun({ct_erl_args,_}) -> false; + (_) -> true end, Init), + + %% convert relative dirs added with pa or pz (pre erl_args on + %% the run_test command line) to absolute so that app modules + %% can be found even after CT changes CWD to logdir + rel_to_abs(CtArgs), + + Args = + case application:get_env(common_test, run_test_start_opts) of + {ok,EnvStartOpts} -> + FlagFilter = fun(Flags) -> + lists:filter(fun({root,_}) -> false; + ({progname,_}) -> false; + ({home,_}) -> false; + ({noshell,_}) -> false; + ({noinput,_}) -> false; + (_) -> true + end, Flags) + end, + %% used for purpose of testing the run_test interface + io:format(user, "~n-------------------- START ARGS --------------------~n", []), + io:format(user, "--- Init args:~n~p~n", [FlagFilter(Init)]), + io:format(user, "--- CT args:~n~p~n", [FlagFilter(CtArgs)]), + EnvArgs = opts2args(EnvStartOpts), + io:format(user, "--- Env opts -> args:~n~p~n =>~n~p~n", + [EnvStartOpts,EnvArgs]), + Merged = merge_arguments(CtArgs ++ EnvArgs), + io:format(user, "--- Merged args:~n~p~n", [FlagFilter(Merged)]), + io:format(user, "----------------------------------------------------~n~n", []), + Merged; + _ -> + merge_arguments(CtArgs) + end, + case proplists:get_value(help, Args) of + undefined -> script_start(Args); + _ -> script_usage() + end. + +script_start(Args) -> Tracing = start_trace(Args), - Res = + Res = case ct_repeat:loop_test(script, Args) of - false -> + false -> {ok,Cwd} = file:get_cwd(), - CTVsn = + CTVsn = case filename:basename(code:lib_dir(common_test)) of CTBase when is_list(CTBase) -> case string:tokens(CTBase, "-") of @@ -76,7 +132,7 @@ script_start() -> io:format("~nCommon Test~s starting (cwd is ~s)~n~n", [CTVsn,Cwd]), Self = self(), Pid = spawn_link(fun() -> script_start1(Self, Args) end), - receive + receive {'EXIT',Pid,Reason} -> case Reason of {user_error,What} -> @@ -98,296 +154,335 @@ script_start() -> Result end, stop_trace(Tracing), + timer:sleep(1000), Res. script_start1(Parent, Args) -> - case lists:keymember(preload, 1, Args) of - true -> preload(); - false -> ok - end, - - VtsOrShell = - case lists:keymember(vts, 1, Args) of - true -> - vts; - false -> - case lists:keymember(shell, 1, Args) of - true -> shell; - false -> false - end - end, - LogDir = - case lists:keysearch(logdir, 1, Args) of - {value,{logdir,[LogD]}} -> LogD; - false -> "." - end, - EvHandlers = - case lists:keysearch(event_handler, 1, Args) of - {value,{event_handler,Handlers}} -> - lists:map(fun(H) -> {list_to_atom(H),[]} end, Handlers); - false -> - [] - end, - Cover = - case lists:keysearch(cover, 1, Args) of - {value,{cover,CoverFile}} -> - {cover,?abs(CoverFile)}; - false -> - false - end, - - case lists:keysearch(ct_decrypt_key, 1, Args) of - {value,{_,[DecryptKey]}} -> + %% read general start flags + Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args), + Vts = get_start_opt(vts, true, Args), + Shell = get_start_opt(shell, true, Args), + Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), + LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), + MultTT = get_start_opt(multiply_timetraps, + fun([MT]) -> list_to_integer(MT) end, 1, Args), + ScaleTT = get_start_opt(scale_timetraps, + fun([CT]) -> list_to_atom(CT); + ([]) -> true + end, false, Args), + EvHandlers = event_handler_args2opts(Args), + + %% check flags and set corresponding application env variables + + %% ct_decrypt_key | ct_decrypt_file + case proplists:get_value(ct_decrypt_key, Args) of + [DecryptKey] -> application:set_env(common_test, decrypt, {key,DecryptKey}); - false -> - case lists:keysearch(ct_decrypt_file, 1, Args) of - {value,{_,[DecryptFile]}} -> - application:set_env(common_test, decrypt, - {file,filename:absname(DecryptFile)}); - false -> + undefined -> + case proplists:get_value(ct_decrypt_file, Args) of + [DecryptFile] -> + application:set_env(common_test, decrypt, + {file,?abs(DecryptFile)}); + undefined -> application:unset_env(common_test, decrypt) end end, - - case lists:keysearch(no_auto_compile, 1, Args) of - {value,_} -> - application:set_env(common_test, auto_compile, false); - false -> - application:set_env(common_test, auto_compile, true), - - InclDirs = - case lists:keysearch(include,1,Args) of - {value,{include,Incl}} when is_list(hd(Incl)) -> - Incl; - {value,{include,Incl}} when is_list(Incl) -> - [Incl]; + %% no_auto_compile + include + IncludeDirs = + case proplists:get_value(no_auto_compile, Args) of + undefined -> + application:set_env(common_test, auto_compile, true), + InclDirs = + case proplists:get_value(include, Args) of + Incl when is_list(hd(Incl)) -> + Incl; + Incl when is_list(Incl) -> + [Incl]; + undefined -> + [] + end, + case os:getenv("CT_INCLUDE_PATH") of false -> - [] - end, - case os:getenv("CT_INCLUDE_PATH") of - false -> - application:set_env(common_test, include, InclDirs); - CtInclPath -> - InclDirs1 = string:tokens(CtInclPath,[$:,$ ,$,]), - application:set_env(common_test, include, InclDirs1++InclDirs) - end - end, - - case lists:keysearch(basic_html, 1, Args) of - {value,_} -> - application:set_env(common_test, basic_html, true); - false -> - application:set_env(common_test, basic_html, false) - end, - - Result = - case lists:keysearch(refresh_logs, 1, Args) of - {value,{refresh_logs,Refresh}} -> - LogDir1 = case Refresh of - [] -> LogDir; - [RefreshDir] -> ?abs(RefreshDir) - end, - {ok,Cwd} = file:get_cwd(), - file:set_cwd(LogDir1), - timer:sleep(500), % give the shell time to print version etc - io:nl(), - case catch ct_logs:make_all_suites_index(refresh) of - {'EXIT',ASReason} -> - file:set_cwd(Cwd), - {error,{all_suites_index,ASReason}}; - _ -> - case catch ct_logs:make_all_runs_index(refresh) of - {'EXIT',ARReason} -> - file:set_cwd(Cwd), - {error,{all_runs_index,ARReason}}; - _ -> - file:set_cwd(Cwd), - io:format("Logs in ~s refreshed!~n~n", [LogDir1]), - timer:sleep(500), % time to flush io before quitting - ok - end + application:set_env(common_test, include, InclDirs), + InclDirs; + CtInclPath -> + AllInclDirs = + string:tokens(CtInclPath,[$:,$ ,$,]) ++ InclDirs, + application:set_env(common_test, include, AllInclDirs), + AllInclDirs end; - false -> - case lists:keysearch(ct_config, 1, Args) of - {value,{ct_config,ConfigFiles}} -> - case lists:keysearch(spec, 1, Args) of - false -> - case get_configfiles(ConfigFiles, [], LogDir, - EvHandlers) of - ok -> - script_start2(VtsOrShell, ConfigFiles, - EvHandlers, Args, LogDir, - Cover); - Error -> - Error - end; - _ -> - script_start2(VtsOrShell, ConfigFiles, - EvHandlers, Args, LogDir, Cover) - end; - false -> - case install([{config,[]}, - {event_handler,EvHandlers}], - LogDir) of - ok -> - script_start2(VtsOrShell, [], EvHandlers, - Args, LogDir, Cover); - Error -> - Error - end - end + _ -> + application:set_env(common_test, auto_compile, false), + [] end, + %% silent connections + SilentConns = + get_start_opt(silent_connections, + fun(["all"]) -> []; + (Conns) -> [list_to_atom(Conn) || Conn <- Conns] + end, Args), + %% stylesheet + Stylesheet = get_start_opt(stylesheet, + fun([SS]) -> ?abs(SS) end, Args), + %% basic_html - used by ct_logs + case proplists:get_value(basic_html, Args) of + undefined -> + application:set_env(common_test, basic_html, false); + _ -> + application:set_env(common_test, basic_html, true) + end, + + StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover, + logdir = LogDir, event_handlers = EvHandlers, + include = IncludeDirs, + silent_connections = SilentConns, + stylesheet = Stylesheet, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT}, + + %% check if log files should be refreshed or go on to run tests... + Result = run_or_refresh(StartOpts, Args), + %% send final results to starting process waiting in script_start/0 Parent ! {self(), Result}. -get_configfiles([File|Files], Acc, LogDir, EvHandlers) -> - case filelib:is_file(File) of - true -> - get_configfiles(Files, [?abs(File)|Acc], - LogDir, EvHandlers); - false -> - {error,{cant_read_config_file,File}} - end; -get_configfiles([], Acc, LogDir, EvHandlers) -> - install([{config,lists:reverse(Acc)}, {event_handler,EvHandlers}], LogDir). +run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> + case proplists:get_value(refresh_logs, Args) of + undefined -> + script_start2(StartOpts, Args); + Refresh -> + LogDir1 = case Refresh of + [] -> which(logdir,LogDir); + [RefreshDir] -> ?abs(RefreshDir) + end, + {ok,Cwd} = file:get_cwd(), + file:set_cwd(LogDir1), + %% give the shell time to print version etc + timer:sleep(500), + io:nl(), + case catch ct_logs:make_all_suites_index(refresh) of + {'EXIT',ASReason} -> + file:set_cwd(Cwd), + {error,{all_suites_index,ASReason}}; + _ -> + case catch ct_logs:make_all_runs_index(refresh) of + {'EXIT',ARReason} -> + file:set_cwd(Cwd), + {error,{all_runs_index,ARReason}}; + _ -> + file:set_cwd(Cwd), + io:format("Logs in ~s refreshed!~n~n", [LogDir1]), + timer:sleep(500), % time to flush io before quitting + ok + end + end + end. -script_start2(false, ConfigFiles, EvHandlers, Args, LogDir, Cover) -> - case lists:keysearch(spec, 1, Args) of - {value,{spec,[]}} -> +script_start2(StartOpts = #opts{vts = undefined, + shell = undefined}, Args) -> + TestSpec = proplists:get_value(spec, Args), + {Terms,Opts} = + case TestSpec of + Specs when Specs =/= [], Specs =/= undefined -> + %% using testspec as input for test + Relaxed = get_start_opt(allow_user_terms, true, false, Args), + case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of + {E,Reason} when E == error ; E == 'EXIT' -> + {{error,Reason},StartOpts}; + TS -> + SpecStartOpts = get_data_for_node(TS, node()), + + Label = choose_val(StartOpts#opts.label, + SpecStartOpts#opts.label), + + LogDir = choose_val(StartOpts#opts.logdir, + SpecStartOpts#opts.logdir), + + Cover = choose_val(StartOpts#opts.cover, + SpecStartOpts#opts.cover), + MultTT = choose_val(StartOpts#opts.multiply_timetraps, + SpecStartOpts#opts.multiply_timetraps), + ScaleTT = choose_val(StartOpts#opts.scale_timetraps, + SpecStartOpts#opts.scale_timetraps), + AllEvHs = merge_vals([StartOpts#opts.event_handlers, + SpecStartOpts#opts.event_handlers]), + AllInclude = merge_vals([StartOpts#opts.include, + SpecStartOpts#opts.include]), + application:set_env(common_test, include, AllInclude), + + {TS,StartOpts#opts{label = Label, + testspecs = Specs, + cover = Cover, + logdir = LogDir, + config = SpecStartOpts#opts.config, + event_handlers = AllEvHs, + include = AllInclude, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT}} + end; + _ -> + {undefined,StartOpts} + end, + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), + case {TestSpec,Terms} of + {_,{error,_}=Error} -> + Error; + {[],_} -> {error,no_testspec_specified}; - {value,{spec,Specs}} -> - Relaxed = lists:keymember(allow_user_terms, 1, Args), - %% using testspec as input for test - case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of - {error,Reason} -> - {error,Reason}; - TS -> - {LogDir1,TSCoverFile,ConfigFiles1,EvHandlers1,Include1} = - get_data_for_node(TS,node()), - UserInclude = - case application:get_env(common_test, include) of - {ok,Include} -> Include++Include1; - _ -> Include1 - end, - application:set_env(common_test, include, UserInclude), - LogDir2 = which_logdir(LogDir,LogDir1), - CoverOpt = case {Cover,TSCoverFile} of - {false,undef} -> []; - {_,undef} -> [Cover]; - {false,_} -> [{cover,TSCoverFile}] - end, - case get_configfiles(ConfigFiles++ConfigFiles1, - [], LogDir2, - EvHandlers++EvHandlers1) of - ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, CoverOpt, Args, LogDir2); - Error -> - Error - end + {undefined,_} -> % no testspec used + case check_and_install_configfiles(InitConfig, TheLogDir, + Opts#opts.event_handlers) of + ok -> % go on read tests from start flags + script_start3(Opts#opts{config=InitConfig, + logdir=TheLogDir}, Args); + Error -> + Error end; - false -> - script_start3(false, ConfigFiles, EvHandlers, Args, LogDir, Cover) + {_,_} -> % testspec used + %% merge config from start flags with config from testspec + AllConfig = merge_vals([InitConfig, Opts#opts.config]), + case check_and_install_configfiles(AllConfig, TheLogDir, + Opts#opts.event_handlers) of + ok -> % read tests from spec + {Run,Skip} = ct_testspec:prepare_tests(Terms, node()), + do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir}, Args); + Error -> + Error + end end; -script_start2(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) -> - script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover). -script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) -> - case lists:keysearch(dir, 1, Args) of - {value,{dir,[]}} -> - {error,no_dir_specified}; - {value,{dir,Dirs}} -> - script_start4(VtsOrShell, ConfigFiles, EvHandlers, tests(Dirs), - Cover, Args, LogDir); +script_start2(StartOpts, Args) -> + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + LogDir = which(logdir, StartOpts#opts.logdir), + case check_and_install_configfiles(InitConfig, LogDir, + StartOpts#opts.event_handlers) of + ok -> % go on read tests from start flags + script_start3(StartOpts#opts{config=InitConfig, + logdir=LogDir}, Args); + Error -> + Error + end. + +check_and_install_configfiles(Configs, LogDir, EvHandlers) -> + case ct_config:check_config_files(Configs) of false -> - case lists:keysearch(suite, 1, Args) of - {value,{suite,[]}} -> + install([{config,Configs}, + {event_handler,EvHandlers}], LogDir); + {value,{error,{nofile,File}}} -> + {error,{cant_read_config_file,File}}; + {value,{error,{wrong_config,Message}}}-> + {error,{wrong_config,Message}}; + {value,{error,{callback,Info}}} -> + {error,{cant_load_callback_module,Info}} + end. + +script_start3(StartOpts, Args) -> + case proplists:get_value(dir, Args) of + [] -> + {error,no_dir_specified}; + Dirs when is_list(Dirs) -> + script_start4(StartOpts#opts{tests = tests(Dirs)}, Args); + undefined -> + case proplists:get_value(suite, Args) of + [] -> {error,no_suite_specified}; - {value,{suite,Suites}} -> - StepOrCover = - case lists:keysearch(step, 1, Args) of - {value,Step} -> Step; - false -> Cover - end, - S2M = fun(S) -> - {filename:dirname(S), - list_to_atom( - filename:rootname(filename:basename(S)))} - end, - DirMods = lists:map(S2M, Suites), - {Specified,GroupsAndCases} = - case {lists:keysearch(group, 1, Args), - lists:keysearch('case', 1, Args)} of - {{value,{_,Gs}},{value,{_,Cs}}} -> {true,Gs++Cs}; - {{value,{_,Gs}},_} -> {true,Gs}; - {_,{value,{_,Cs}}} -> {true,Cs}; - _ -> {false,[]} - end, - if Specified, length(GroupsAndCases) == 0 -> - {error,no_case_or_group_specified}; - Specified, length(DirMods) > 1 -> + Suites when is_list(Suites) -> + StartOpts1 = + get_start_opt(step, + fun(Step) -> + StartOpts#opts{step = Step, + cover = undefined} + end, StartOpts, Args), + DirMods = [suite_to_test(S) || S <- Suites], + case groups_and_cases(proplists:get_value(group, Args), + proplists:get_value(testcase, Args)) of + Error = {error,_} -> + Error; + [] when DirMods =/= [] -> + Ts = tests(DirMods), + script_start4(StartOpts1#opts{tests = Ts}, Args); + GroupsAndCases when length(DirMods) == 1 -> + Ts = tests(DirMods, GroupsAndCases), + script_start4(StartOpts1#opts{tests = Ts}, Args); + [_,_|_] when length(DirMods) > 1 -> {error,multiple_suites_and_cases}; - length(GroupsAndCases) > 0, length(DirMods) == 1 -> - GsAndCs = lists:map(fun(C) -> list_to_atom(C) end, - GroupsAndCases), - script_start4(VtsOrShell, ConfigFiles, EvHandlers, - tests(DirMods, GsAndCs), - StepOrCover, Args, LogDir); - not Specified, length(DirMods) > 0 -> - script_start4(VtsOrShell, ConfigFiles, EvHandlers, - tests(DirMods), - StepOrCover, Args, LogDir); - true -> - {error,incorrect_suite_and_case_options} + _ -> + {error,incorrect_suite_option} end; - false when VtsOrShell=/=false -> - script_start4(VtsOrShell, ConfigFiles, EvHandlers, - [], Cover, Args, LogDir); - false -> - script_usage(), - {error,incorrect_usage} + undefined -> + if StartOpts#opts.vts ; StartOpts#opts.shell -> + script_start4(StartOpts#opts{tests = []}, Args); + true -> + script_usage(), + {error,incorrect_usage} + end end end. -script_start4(vts, ConfigFiles, EvHandlers, Tests, false, _Args, LogDir) -> +script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, + tests = Tests, logdir = LogDir}, _Args) -> + ConfigFiles = + lists:foldl(fun({ct_config_plain,CfgFiles}, AllFiles) when + is_list(hd(CfgFiles)) -> + AllFiles ++ CfgFiles; + ({ct_config_plain,CfgFile}, AllFiles) when + is_integer(hd(CfgFile)) -> + AllFiles ++ [CfgFile]; + (_, AllFiles) -> + AllFiles + end, [], Config), vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests); -script_start4(shell, ConfigFiles, EvHandlers, _Tests, false, Args, LogDir) -> - Opts = [{config,ConfigFiles},{event_handler,EvHandlers}], - if ConfigFiles == [] -> + +script_start4(#opts{label = Label, shell = true, config = Config, + event_handlers = EvHandlers, + logdir = LogDir, testspecs = Specs}, _Args) -> + %% label - used by ct_logs + application:set_env(common_test, test_label, Label), + + InstallOpts = [{config,Config},{event_handler,EvHandlers}], + if Config == [] -> ok; true -> - io:format("\nInstalling: ~p\n\n", [ConfigFiles]) + io:format("\nInstalling: ~p\n\n", [Config]) end, - case install(Opts) of + case install(InstallOpts) of ok -> ct_util:start(interactive, LogDir), - log_ts_names(Args), + log_ts_names(Specs), io:nl(), ok; Error -> Error end; -script_start4(vts, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) -> - %% Add support later (maybe). - script_usage(), - erlang:halt(); -script_start4(shell, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) -> - %% Add support later (maybe). - script_usage(); -script_start4(false, _CfgFs, _EvHs, Tests, Cover={cover,_}, Args, LogDir) -> - do_run(Tests, [], [Cover], Args, LogDir); -script_start4(false, _ConfigFiles, _EvHandlers, Tests, false, Args, LogDir) -> - do_run(Tests, [], [], Args, LogDir); -script_start4(false, _ConfigFiles, _EvHandlers, Test, Step, Args, LogDir) -> - do_run(Test, [], [Step], Args, LogDir); -script_start4(vts, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) -> - script_usage(), + +script_start4(#opts{vts = true, cover = Cover}, _) -> + case Cover of + undefined -> + script_usage(); + _ -> + %% Add support later (maybe). + io:format("\nCan't run cover in vts mode.\n\n", []) + end, erlang:halt(); -script_start4(shell, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) -> - script_usage(). + +script_start4(#opts{shell = true, cover = Cover}, _) -> + case Cover of + undefined -> + script_usage(); + _ -> + %% Add support later (maybe). + io:format("\nCan't run cover in interactive mode.\n\n", []) + end; + +script_start4(Opts = #opts{tests = Tests}, Args) -> + do_run(Tests, [], Opts, Args). %%%----------------------------------------------------------------- %%% @spec script_usage() -> ok -%%% @doc Print script usage information for <code>run_test</code>. +%%% @doc Print usage information for <code>run_test</code>. script_usage() -> io:format("\n\nUsage:\n\n"), io:format("Run tests in web based GUI:\n\n" @@ -396,24 +491,29 @@ script_usage() -> "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" "\n\t[-dir TestDir1 TestDir2 .. TestDirN] |" "\n\t[-suite Suite [-case Case]]" - "\n\t[-include InclDir1 InclDir2 .. InclDirN]" + "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" + "\n\t[-multiply_timetraps N]" + "\n\t[-scale_timetraps]" "\n\t[-basic_html]\n\n"), io:format("Run tests from command line:\n\n" "\trun_test [-dir TestDir1 TestDir2 .. TestDirN] |" "\n\t[-suite Suite1 Suite2 .. SuiteN [-case Case1 Case2 .. CaseN]]" "\n\t[-step [config | keep_inactive]]" "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" + "\n\t[-userconfig CallbackModule ConfigFile1 .. ConfigFileN]" "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" "\n\t[-logdir LogDir]" "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" - "\n\t[-include InclDir1 InclDir2 .. InclDirN]" + "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" - "\n\t[-basic_html]" - "\n\t[-repeat N [-force_stop]] |" + "\n\t[-multiply_timetraps N]" + "\n\t[-scale_timetraps]" + "\n\t[-basic_html]" + "\n\t[-repeat N [-force_stop]] |" "\n\t[-duration HHMMSS [-force_stop]] |" "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"), io:format("Run tests using test specification:\n\n" @@ -426,10 +526,12 @@ script_usage() -> "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" - "\n\t[-include InclDir1 InclDir2 .. InclDirN]" + "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" - "\n\t[-basic_html]" - "\n\t[-repeat N [-force_stop]] |" + "\n\t[-multiply_timetraps N]" + "\n\t[-scale_timetraps]" + "\n\t[-basic_html]" + "\n\t[-repeat N [-force_stop]] |" "\n\t[-duration HHMMSS [-force_stop]] |" "\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"), io:format("Refresh the HTML index files:\n\n" @@ -440,7 +542,6 @@ script_usage() -> "\trun_test -shell" "\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]" "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]\n\n"). - %%%----------------------------------------------------------------- %%% @hidden @@ -468,7 +569,7 @@ install(Opts, LogDir) -> [io:format(Fd, "~p.\n", [Opt]) || Opt <- Opts], file:close(Fd), ok; - {error,Reason} -> + {error,Reason} -> io:format("CT failed to install configuration data. Please " "verify that the log directory exists and that " "write permission is set.\n\n", []), @@ -487,69 +588,62 @@ variables_file_name(Dir) -> filename:join(Dir, "variables-"++atom_to_list(node())). %%%----------------------------------------------------------------- -%%% @hidden +%%% @spec run_test(Opts) -> Result +%%% Opts = [tuple()] +%%% Result = [TestResult] | {error,Reason} +%%% +%%% @doc Start tests from the erlang shell or from an erlang program. %%% @equiv ct:run_test/1 +%%%----------------------------------------------------------------- -%% Opts = [OptTuples] -%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} | -%% {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} | -%% {logdir,LogDir} | {cover,CoverSpecFile} | {step,StepOpts} | -%% {silent_connections,Conns} | {event_handler,EventHandlers} | -%% {include,InclDirs} | {auto_compile,Bool} | -%% {repeat,N} | {duration,DurTime} | {until,StopTime} | {force_stop,Bool} | -%% {decrypt,KeyOrFile} - -run_test(Opt) when is_tuple(Opt) -> - run_test([Opt]); - -run_test(Opts) when is_list(Opts) -> - case lists:keysearch(refresh_logs, 1, Opts) of - {value,{_,RefreshDir}} -> - refresh_logs(?abs(RefreshDir)), - ok; - false -> - Tracing = start_trace(Opts), +run_test(StartOpt) when is_tuple(StartOpt) -> + run_test([StartOpt]); + +run_test(StartOpts) when is_list(StartOpts) -> + case proplists:get_value(refresh_logs, StartOpts) of + undefined -> + Tracing = start_trace(StartOpts), {ok,Cwd} = file:get_cwd(), - io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]), + io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]), Res = - case ct_repeat:loop_test(func, Opts) of + case ct_repeat:loop_test(func, StartOpts) of false -> - case catch run_test1(Opts) of - {'EXIT',Reason} -> + case catch run_test1(StartOpts) of + {'EXIT',Reason} -> file:set_cwd(Cwd), {error,Reason}; - Result -> + Result -> Result end; Result -> Result end, stop_trace(Tracing), - Res + Res; + RefreshDir -> + refresh_logs(?abs(RefreshDir)), + ok end. -run_test1(Opts) -> - LogDir = - case lists:keysearch(logdir, 1, Opts) of - {value,{_,LD}} when is_list(LD) -> LD; - false -> "." - end, - CfgFiles = - case lists:keysearch(config, 1, Opts) of - {value,{_,Files=[File|_]}} when is_list(File) -> - Files; - {value,{_,File=[C|_]}} when is_integer(C) -> - [File]; - {value,{_,[]}} -> - []; - false -> - [] - end, +run_test1(StartOpts) -> + %% label + Label = get_start_opt(label, fun(Lbl) when is_list(Lbl) -> Lbl; + (Lbl) when is_atom(Lbl) -> atom_to_list(Lbl) + end, StartOpts), + %% logdir + LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end, + StartOpts), + %% config & userconfig + CfgFiles = ct_config:get_config_file_list(StartOpts), + + %% event handlers EvHandlers = - case lists:keysearch(event_handler, 1, Opts) of - {value,{_,H}} when is_atom(H) -> + case proplists:get_value(event_handler, StartOpts) of + undefined -> + []; + H when is_atom(H) -> [{H,[]}]; - {value,{_,H}} -> + H -> Hs = if is_tuple(H) -> [H]; is_list(H) -> H; @@ -564,41 +658,39 @@ run_test1(Opts) -> {EH,Args}; (_) -> [] - end, Hs)); - _ -> - [] - end, - SilentConns = - case lists:keysearch(silent_connections, 1, Opts) of - {value,{_,all}} -> - []; - {value,{_,Conns}} -> - Conns; - _ -> - undefined - end, - Cover = - case lists:keysearch(cover, 1, Opts) of - {value,{_,CoverFile}} -> - [{cover,?abs(CoverFile)}]; - _ -> - [] + end, Hs)) end, + + %% silent connections + SilentConns = get_start_opt(silent_connections, + fun(all) -> []; + (Conns) -> Conns + end, StartOpts), + %% stylesheet + Stylesheet = get_start_opt(stylesheet, + fun(SS) -> ?abs(SS) end, + StartOpts), + %% code coverage + Cover = get_start_opt(cover, + fun(CoverFile) -> ?abs(CoverFile) end, StartOpts), + + %% timetrap manipulation + MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts), + ScaleTT = get_start_opt(scale_timetraps, value, false, StartOpts), + + %% auto compile & include files Include = - case lists:keysearch(auto_compile, 1, Opts) of - {value,{auto_compile,ACBool}} -> - application:set_env(common_test, auto_compile, ACBool), - []; - _ -> + case proplists:get_value(auto_compile, StartOpts) of + undefined -> application:set_env(common_test, auto_compile, true), InclDirs = - case lists:keysearch(include, 1, Opts) of - {value,{include,Incl}} when is_list(hd(Incl)) -> - Incl; - {value,{include,Incl}} when is_list(Incl) -> - [Incl]; - false -> - [] + case proplists:get_value(include, StartOpts) of + undefined -> + []; + Incl when is_list(hd(Incl)) -> + Incl; + Incl when is_list(Incl) -> + [Incl] end, case os:getenv("CT_INCLUDE_PATH") of false -> @@ -609,117 +701,173 @@ run_test1(Opts) -> AllInclDirs = InclDirs1++InclDirs, application:set_env(common_test, include, AllInclDirs), AllInclDirs - end + end; + ACBool -> + application:set_env(common_test, auto_compile, ACBool), + [] end, - case lists:keysearch(decrypt, 1, Opts) of - {value,{_,Key={key,_}}} -> + %% decrypt config file + case proplists:get_value(decrypt, StartOpts) of + undefined -> + application:unset_env(common_test, decrypt); + Key={key,_} -> application:set_env(common_test, decrypt, Key); - {value,{_,{file,KeyFile}}} -> - application:set_env(common_test, decrypt, {file,filename:absname(KeyFile)}); - false -> - application:unset_env(common_test, decrypt) + {file,KeyFile} -> + application:set_env(common_test, decrypt, {file,?abs(KeyFile)}) end, - case lists:keysearch(basic_html, 1, Opts) of - {value,{basic_html,BasicHtmlBool}} -> - application:set_env(common_test, basic_html, BasicHtmlBool); - _ -> - application:set_env(common_test, basic_html, false) + %% basic html - used by ct_logs + case proplists:get_value(basic_html, StartOpts) of + undefined -> + application:set_env(common_test, basic_html, false); + BasicHtmlBool -> + application:set_env(common_test, basic_html, BasicHtmlBool) end, - case lists:keysearch(spec, 1, Opts) of - {value,{_,Specs}} -> - Relaxed = - case lists:keysearch(allow_user_terms, 1, Opts) of - {value,{_,true}} -> true; - _ -> false - end, + %% stepped execution + Step = get_start_opt(step, value, StartOpts), + + Opts = #opts{label = Label, + cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, + event_handlers = EvHandlers, include = Include, + silent_connections = SilentConns, + stylesheet = Stylesheet, + multiply_timetraps = MultiplyTT, + scale_timetraps = ScaleTT}, + + %% test specification + case proplists:get_value(spec, StartOpts) of + undefined -> + case proplists:get_value(prepared_tests, StartOpts) of + undefined -> % use dir|suite|case + run_dir(Opts, StartOpts); + {{Run,Skip},Specs} -> % use prepared tests + run_prepared(Run, Skip, Opts#opts{testspecs = Specs}, StartOpts) + end; + Specs -> + Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts), %% using testspec(s) as input for test - run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover, - replace_opt([{silent_connections,SilentConns}], Opts)); - false -> - case lists:keysearch(prepared_tests, 1, Opts) of - {value,{_,{Run,Skip},Specs}} -> % use prepared tests - run_prepared(LogDir, CfgFiles, EvHandlers, - Run, Skip, Cover, - replace_opt([{silent_connections,SilentConns}, - {spec,Specs}],Opts)); - false -> % use dir|suite|case - StepOrCover = - case lists:keysearch(step, 1, Opts) of - {value,Step} -> [Step]; - false -> Cover - end, - run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, - replace_opt([{silent_connections,SilentConns}], Opts)) - end + run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts) end. -replace_opt([O={Key,_Val}|Os], Opts) -> - [O | replace_opt(Os, lists:keydelete(Key, 1, Opts))]; -replace_opt([], Opts) -> - Opts. - -run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover, Opts) -> +run_spec_file(Relaxed, + Opts = #opts{testspecs = Specs, config = CfgFiles}, + StartOpts) -> Specs1 = case Specs of [X|_] when is_integer(X) -> [Specs]; _ -> Specs end, - AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), + AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), log_ts_names(AbsSpecs), case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of - {error,CTReason} -> + {Error,CTReason} when Error == error ; Error == 'EXIT' -> exit(CTReason); TS -> - {LogDir1,TSCoverFile,CfgFiles1,EvHandlers1,Include1} = - get_data_for_node(TS, node()), - application:set_env(common_test, include, Include++Include1), - LogDir2 = which_logdir(LogDir, LogDir1), - CoverOpt = case {Cover,TSCoverFile} of - {[],undef} -> []; - {_,undef} -> Cover; - {[],_} -> [{cover,TSCoverFile}] - end, - case get_configfiles(CfgFiles++CfgFiles1, [], LogDir2, - EvHandlers++EvHandlers1) of + SpecOpts = get_data_for_node(TS, node()), + Label = choose_val(Opts#opts.label, + SpecOpts#opts.label), + LogDir = choose_val(Opts#opts.logdir, + SpecOpts#opts.logdir), + AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), + Cover = choose_val(Opts#opts.cover, + SpecOpts#opts.cover), + MultTT = choose_val(Opts#opts.multiply_timetraps, + SpecOpts#opts.multiply_timetraps), + ScaleTT = choose_val(Opts#opts.scale_timetraps, + SpecOpts#opts.scale_timetraps), + AllEvHs = merge_vals([Opts#opts.event_handlers, + SpecOpts#opts.event_handlers]), + AllInclude = merge_vals([Opts#opts.include, + SpecOpts#opts.include]), + application:set_env(common_test, include, AllInclude), + + case check_and_install_configfiles(AllConfig, + which(logdir,LogDir), + AllEvHs) of ok -> + Opts1 = Opts#opts{label = Label, + cover = Cover, + logdir = which(logdir, LogDir), + config = AllConfig, + event_handlers = AllEvHs, + include = AllInclude, + testspecs = AbsSpecs, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, CoverOpt, - replace_opt([{spec,AbsSpecs}], Opts), - LogDir2); + reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); {error,GCFReason} -> exit(GCFReason) end end. -run_prepared(LogDir, CfgFiles, EvHandlers, Run, Skip, Cover, Opts) -> - case get_configfiles(CfgFiles, [], LogDir, EvHandlers) of +run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, + config = CfgFiles, + event_handlers = EvHandlers}, + StartOpts) -> + LogDir1 = which(logdir, LogDir), + case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of ok -> - do_run(Run, Skip, Cover, Opts, LogDir); + reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1}, + StartOpts)); {error,Reason} -> exit(Reason) - end. - -run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) -> - AbsCfgFiles = - lists:map(fun(F) -> - AbsName = ?abs(F), - case filelib:is_file(AbsName) of - true -> AbsName; - false -> exit({no_such_file,AbsName}) - end - end, CfgFiles), - - case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir) of + end. + +check_config_file(Callback, File)-> + case code:is_loaded(Callback) of + false -> + case code:load_file(Callback) of + {module,_} -> ok; + {error,Why} -> exit({cant_load_callback_module,Why}) + end; + _ -> + ok + end, + case Callback:check_parameter(File) of + {ok,{file,File}}-> + ?abs(File); + {ok,{config,_}}-> + File; + {error,{wrong_config,Message}}-> + exit({wrong_config,{Callback,Message}}); + {error,{nofile,File}}-> + exit({no_such_file,?abs(File)}) + end. + +run_dir(Opts = #opts{logdir = LogDir, + config = CfgFiles, + event_handlers = EvHandlers}, StartOpts) -> + LogDir1 = which(logdir, LogDir), + Opts1 = Opts#opts{logdir = LogDir1}, + AbsCfgFiles = + lists:map(fun({Callback,FileList})-> + case code:is_loaded(Callback) of + {file,_Path}-> + ok; + false -> + case code:load_file(Callback) of + {module,Callback}-> + ok; + {error,_}-> + exit({no_such_module,Callback}) + end + end, + {Callback, + lists:map(fun(File)-> + check_config_file(Callback, File) + end, FileList)} + end, CfgFiles), + case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir1) of ok -> ok; {error,IReason} -> exit(IReason) end, - case lists:keysearch(dir,1,Opts) of + case lists:keysearch(dir, 1, StartOpts) of {value,{_,Dirs=[Dir|_]}} when not is_integer(Dir), length(Dirs)>1 -> %% multiple dirs (no suite) - do_run(tests(Dirs), [], StepOrCover, Opts, LogDir); + reformat_result(catch do_run(tests(Dirs), [], Opts1, StartOpts)); false -> % no dir %% fun for converting suite name to {Dir,Mod} tuple S2M = fun(S) when is_list(S) -> @@ -728,105 +876,131 @@ run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) -> (A) -> {".",A} end, - case lists:keysearch(suite, 1, Opts) of + case lists:keysearch(suite, 1, StartOpts) of {value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) -> {Dir,Mod} = S2M(Suite), - case listify(proplists:get_value(group, Opts, [])) ++ - listify(proplists:get_value(testcase, Opts, [])) of + case listify(proplists:get_value(group, StartOpts, [])) ++ + listify(proplists:get_value(testcase, StartOpts, [])) of [] -> - do_run(tests(Dir, listify(Mod)), [], StepOrCover, Opts, LogDir); + reformat_result(catch do_run(tests(Dir, listify(Mod)), + [], Opts1, StartOpts)); GsAndCs -> - do_run(tests(Dir, Mod, GsAndCs), [], StepOrCover, Opts, LogDir) + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)) end; {value,{_,Suites}} -> - do_run(tests(lists:map(S2M, Suites)), [], StepOrCover, Opts, LogDir); + reformat_result(catch do_run(tests(lists:map(S2M, Suites)), + [], Opts1, StartOpts)); _ -> exit(no_tests_specified) - end; + end; {value,{_,Dir}} -> - case lists:keysearch(suite, 1, Opts) of + case lists:keysearch(suite, 1, StartOpts) of {value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) -> - Mod = if is_atom(Suite) -> Suite; - true -> list_to_atom(Suite) + Mod = if is_atom(Suite) -> Suite; + true -> list_to_atom(Suite) end, - case listify(proplists:get_value(group, Opts, [])) ++ - listify(proplists:get_value(testcase, Opts, [])) of + case listify(proplists:get_value(group, StartOpts, [])) ++ + listify(proplists:get_value(testcase, StartOpts, [])) of [] -> - do_run(tests(Dir, listify(Mod)), [], StepOrCover, Opts, LogDir); + reformat_result(catch do_run(tests(Dir, listify(Mod)), + [], Opts1, StartOpts)); GsAndCs -> - do_run(tests(Dir, Mod, GsAndCs), [], StepOrCover, Opts, LogDir) + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)) end; {value,{_,Suites=[Suite|_]}} when is_list(Suite) -> Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites), - do_run(tests(delistify(Dir), Mods), [], StepOrCover, Opts, LogDir); + reformat_result(catch do_run(tests(delistify(Dir), Mods), + [], Opts1, StartOpts)); {value,{_,Suites}} -> - do_run(tests(delistify(Dir), Suites), [], StepOrCover, Opts, LogDir); + reformat_result(catch do_run(tests(delistify(Dir), Suites), + [], Opts1, StartOpts)); false -> % no suite, only dir - do_run(tests(listify(Dir)), [], StepOrCover, Opts, LogDir) - end + reformat_result(catch do_run(tests(listify(Dir)), + [], Opts1, StartOpts)) + end end. %%%----------------------------------------------------------------- -%%% @hidden +%%% @spec run_testspec(TestSpec) -> Result +%%% TestSpec = [term()] %%% +%%% @doc Run test specified by <code>TestSpec</code>. The terms are +%%% the same as those used in test specification files. +%%% @equiv ct:run_testspec/1 +%%%----------------------------------------------------------------- -%% using testspec(s) as input for test run_testspec(TestSpec) -> {ok,Cwd} = file:get_cwd(), io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]), case catch run_testspec1(TestSpec) of - {'EXIT',Reason} -> + {'EXIT',Reason} -> file:set_cwd(Cwd), {error,Reason}; - Result -> + Result -> Result end. run_testspec1(TestSpec) -> - case ct_testspec:collect_tests_from_list(TestSpec,false) of - {error,CTReason} -> + case catch ct_testspec:collect_tests_from_list(TestSpec, false) of + {E,CTReason} when E == error ; E == 'EXIT' -> exit(CTReason); TS -> - {LogDir,TSCoverFile,CfgFiles,EvHandlers,Include} = - get_data_for_node(TS,node()), - case os:getenv("CT_INCLUDE_PATH") of - false -> - application:set_env(common_test, include, Include); - CtInclPath -> - EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]), - application:set_env(common_test, include, EnvInclude++Include) - end, - CoverOpt = if TSCoverFile == undef -> []; - true -> [{cover,TSCoverFile}] - end, - case get_configfiles(CfgFiles,[],LogDir,EvHandlers) of + Opts = get_data_for_node(TS, node()), + + AllInclude = + case os:getenv("CT_INCLUDE_PATH") of + false -> + Opts#opts.include; + CtInclPath -> + EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]), + EnvInclude++Opts#opts.include + end, + application:set_env(common_test, include, AllInclude), + LogDir1 = which(logdir,Opts#opts.logdir), + case check_and_install_configfiles(Opts#opts.config, LogDir1, + Opts#opts.event_handlers) of ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS,node()), - do_run(Run,Skip,CoverOpt,[],LogDir); + Opts1 = Opts#opts{testspecs = [], + logdir = LogDir1, + include = AllInclude}, + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + reformat_result(catch do_run(Run, Skip, Opts1, [])); {error,GCFReason} -> exit(GCFReason) end end. - -get_data_for_node(#testspec{logdir=LogDirs, - cover=CoverFs, - config=Cfgs, - event_handler=EvHs, - include=Incl}, Node) -> - LogDir = case lists:keysearch(Node,1,LogDirs) of - {value,{Node,Dir}} -> Dir; - false -> "." +get_data_for_node(#testspec{label = Labels, + logdir = LogDirs, + cover = CoverFs, + config = Cfgs, + userconfig = UsrCfgs, + event_handler = EvHs, + include = Incl, + multiply_timetraps = MTs, + scale_timetraps = STs}, Node) -> + Label = proplists:get_value(Node, Labels), + LogDir = case proplists:get_value(Node, LogDirs) of + undefined -> "."; + Dir -> Dir end, - Cover = case lists:keysearch(Node,1,CoverFs) of - {value,{Node,CovFile}} -> CovFile; - false -> undef - end, - ConfigFiles = [F || {N,F} <- Cfgs, N==Node], + Cover = proplists:get_value(Node, CoverFs), + MT = proplists:get_value(Node, MTs), + ST = proplists:get_value(Node, STs), + ConfigFiles = [{?ct_config_txt,F} || {N,F} <- Cfgs, N==Node] ++ + [CBF || {N,CBF} <- UsrCfgs, N==Node], EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node], Include = [I || {N,I} <- Incl, N==Node], - {LogDir,Cover,ConfigFiles,EvHandlers,Include}. - + #opts{label = Label, + logdir = LogDir, + cover = Cover, + config = ConfigFiles, + event_handlers = EvHandlers, + include = Include, + multiply_timetraps = MT, + scale_timetraps = ST}. refresh_logs(LogDir) -> {ok,Cwd} = file:get_cwd(), @@ -851,11 +1025,27 @@ refresh_logs(LogDir) -> end end. -which_logdir(".",Dir) -> +which(logdir, undefined) -> + "."; +which(logdir, Dir) -> Dir; -which_logdir(Dir,_) -> - Dir. - +which(multiply_timetraps, undefined) -> + 1; +which(multiply_timetraps, MT) -> + MT; +which(scale_timetraps, undefined) -> + false; +which(scale_timetraps, ST) -> + ST. + +choose_val(undefined, V1) -> + V1; +choose_val(V0, _V1) -> + V0. + +merge_vals(Vs) -> + lists:append(Vs). + listify([C|_]=Str) when is_integer(C) -> [Str]; listify(L) when is_list(L) -> L; listify(E) -> [E]. @@ -869,22 +1059,43 @@ delistify(E) -> E. %%% @equiv ct:run/3 run(TestDir, Suite, Cases) -> install([]), - do_run(tests(TestDir, Suite, Cases), []). + reformat_result(catch do_run(tests(TestDir, Suite, Cases), [])). %%%----------------------------------------------------------------- %%% @hidden %%% @equiv ct:run/2 run(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) -> install([]), - do_run(tests(TestDir, Suite), []). + reformat_result(catch do_run(tests(TestDir, Suite), [])). %%%----------------------------------------------------------------- %%% @hidden %%% @equiv ct:run/1 run(TestDirs) -> install([]), - do_run(tests(TestDirs), []). + reformat_result(catch do_run(tests(TestDirs), [])). + +reformat_result({user_error,Reason}) -> + {error,Reason}; +reformat_result(Result) -> + Result. +suite_to_test(Suite) -> + {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}. + +groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and + ((Cs == undefined) or (Cs == [])) -> + []; +groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] -> + [list_to_atom(C) || C <- Cs]; +groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] -> + [{list_to_atom(G),all} || G <- Gs]; +groups_and_cases([G], Cs) -> + [{list_to_atom(G),[list_to_atom(C) || C <- Cs]}]; +groups_and_cases([_,_|_] , Cs) when Cs =/= [] -> + {error,multiple_groups_and_cases}; +groups_and_cases(_Gs, _Cs) -> + {error,incorrect_group_or_case_option}. tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) -> [{?testdir(TestDir,Suites),ensure_atom(Suites),all}]; @@ -901,30 +1112,52 @@ tests(TestDir) when is_list(TestDir), is_integer(hd(TestDir)) -> tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) -> [{?testdir(TestDir,all),all,all} || TestDir <- TestDirs]. -do_run(Tests, Opt) -> - do_run(Tests, [], Opt, [], "."). +do_run(Tests, Misc) when is_list(Misc) -> + do_run(Tests, Misc, "."). -do_run(Tests, Opt, LogDir) -> - do_run(Tests, [], Opt, [], LogDir). +do_run(Tests, Misc, LogDir) when is_list(Misc) -> + Opts = + case proplists:get_value(step, Misc) of + undefined -> + #opts{}; + StepOpts -> + #opts{step = StepOpts} + end, + Opts1 = + case proplists:get_value(cover, Misc) of + undefined -> + Opts; + CoverFile -> + Opts#opts{cover = CoverFile} + end, + do_run(Tests, [], Opts1#opts{logdir = LogDir}, []). + +do_run(Tests, Skip, Opts, Args) -> + #opts{label = Label, cover = Cover} = Opts, + + %% label - used by ct_logs + TestLabel = + if Label == undefined -> undefined; + is_atom(Label) -> atom_to_list(Label); + is_list(Label) -> Label; + true -> undefined + end, + application:set_env(common_test, test_label, TestLabel), -do_run(Tests, Skip, Opt, Args, LogDir) -> case code:which(test_server) of non_existing -> exit({error,no_path_to_test_server}); _ -> - Opt1 = - case lists:keysearch(cover, 1, Opt) of - {value,{_,CoverFile}} -> - case ct_cover:get_spec(CoverFile) of - {error,Reason} -> - exit({error,Reason}); - Spec -> - [{cover_spec,Spec} | - lists:keydelete(cover, 1, Opt)] - end; - _ -> - Opt - end, + Opts1 = if Cover == undefined -> + Opts; + true -> + case ct_cover:get_spec(Cover) of + {error,Reason} -> + exit({error,Reason}); + CoverSpec -> + Opts#opts{coverspec = CoverSpec} + end + end, %% This env variable is used by test_server to determine %% which framework it runs under. case os:getenv("TEST_SERVER_FRAMEWORK") of @@ -935,60 +1168,58 @@ do_run(Tests, Skip, Opt, Args, LogDir) -> Other -> erlang:display(list_to_atom("Note: TEST_SERVER_FRAMEWORK = " ++ Other)) end, - case ct_util:start(LogDir) of + case ct_util:start(Opts#opts.logdir) of {error,interactive_mode} -> io:format("CT is started in interactive mode. " "To exit this mode, run ct:stop_interactive().\n" "To enter the interactive mode again, " "run ct:start_interactive()\n\n",[]), {error,interactive_mode}; - _Pid -> - %% save style sheet info - case lists:keysearch(stylesheet, 1, Args) of - {value,{_,SSFile}} -> - ct_util:set_testdata({stylesheet,SSFile}); - _ -> - ct_util:set_testdata({stylesheet,undefined}) - end, - - case lists:keysearch(silent_connections, 1, Args) of - {value,{silent_connections,undefined}} -> - ok; - {value,{silent_connections,[]}} -> + %% save stylesheet info + ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), + %% enable silent connections + case Opts#opts.silent_connections of + [] -> Conns = ct_util:override_silence_all_connections(), ct_logs:log("Silent connections", "~p", [Conns]); - {value,{silent_connections,Cs}} -> - Conns = lists:map(fun(S) when is_list(S) -> - list_to_atom(S); - (A) -> A - end, Cs), + Conns when is_list(Conns) -> ct_util:override_silence_connections(Conns), ct_logs:log("Silent connections", "~p", [Conns]); _ -> ok end, - log_ts_names(Args), + log_ts_names(Opts1#opts.testspecs), TestSuites = suite_tuples(Tests), - {SuiteMakeErrors,AllMakeErrors} = + {_TestSuites1,SuiteMakeErrors,AllMakeErrors} = case application:get_env(common_test, auto_compile) of {ok,false} -> - SuitesNotFound = verify_suites(TestSuites), - {SuitesNotFound,SuitesNotFound}; + {TestSuites1,SuitesNotFound} = + verify_suites(TestSuites), + {TestSuites1,SuitesNotFound,SuitesNotFound}; _ -> {SuiteErrs,HelpErrs} = auto_compile(TestSuites), - {SuiteErrs,SuiteErrs++HelpErrs} + {TestSuites,SuiteErrs,SuiteErrs++HelpErrs} end, case continue(AllMakeErrors) of true -> SavedErrors = save_make_errors(SuiteMakeErrors), ct_repeat:log_loop_info(Args), - {Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors), - R = do_run_test(Tests1, Skip1, Opt1), - ct_util:stop(normal), - R; + + {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors), + + R = (catch do_run_test(Tests1, Skip1, Opts1)), + case R of + {EType,_} = Error when EType == user_error ; + EType == error -> + ct_util:stop(clean), + exit(Error); + _ -> + ct_util:stop(normal), + R + end; false -> io:nl(), ct_util:stop(clean), @@ -1012,7 +1243,7 @@ auto_compile(TestSuites) -> case application:get_env(common_test, include) of {ok,UserInclDirs} when length(UserInclDirs) > 0 -> io:format("Including the following directories:~n"), - [begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end || + [begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end || UserInclDir <- UserInclDirs]; _ -> [] @@ -1020,11 +1251,11 @@ auto_compile(TestSuites) -> SuiteMakeErrors = lists:flatmap(fun({TestDir,Suite} = TS) -> case run_make(suites, TestDir, Suite, UserInclude) of - {error,{make_failed,Bad}} -> + {error,{make_failed,Bad}} -> [{TS,Bad}]; - {error,_} -> + {error,_} -> [{TS,[filename:join(TestDir,"*_SUITE")]}]; - _ -> + _ -> [] end end, TestSuites), @@ -1048,39 +1279,63 @@ auto_compile(TestSuites) -> true -> % already visited {Done,Failed} end - end, {[],[]}, TestSuites), + end, {[],[]}, TestSuites), {SuiteMakeErrors,lists:reverse(HelpMakeErrors)}. %% verify that specified test suites exist (if auto compile is disabled) verify_suites(TestSuites) -> io:nl(), - Verify = - fun({Dir,Suite},NotFound) -> + Verify = + fun({Dir,Suite}=DS,{Found,NotFound}) -> case locate_test_dir(Dir, Suite) of {ok,TestDir} -> if Suite == all -> - NotFound; + {[DS|Found],NotFound}; true -> - Beam = filename:join(TestDir, atom_to_list(Suite)++".beam"), + Beam = filename:join(TestDir, + atom_to_list(Suite)++".beam"), case filelib:is_regular(Beam) of - true -> - NotFound; - false -> - Name = filename:join(TestDir, atom_to_list(Suite)), - io:format("Suite ~w not found in directory ~s~n", - [Suite,TestDir]), - [{{Dir,Suite},[Name]} | NotFound] + true -> + {[DS|Found],NotFound}; + false -> + case code:is_loaded(Suite) of + {file,SuiteFile} -> + %% test suite is already loaded and + %% since auto_compile == false, + %% let's assume the user has + %% loaded the beam file explicitly + ActualDir = filename:dirname(SuiteFile), + {[{ActualDir,Suite}|Found],NotFound}; + false -> + Name = + filename:join(TestDir, + atom_to_list(Suite)), + io:format(user, + "Suite ~w not found" + "in directory ~s~n", + [Suite,TestDir]), + {Found,[{DS,[Name]}|NotFound]} + end end end; {error,_Reason} -> - io:format("Directory ~s is invalid~n", [Dir]), - Name = filename:join(Dir, atom_to_list(Suite)), - [{{Dir,Suite},[Name]} | NotFound] + case code:is_loaded(Suite) of + {file,SuiteFile} -> + %% test suite is already loaded and since + %% auto_compile == false, let's assume the + %% user has loaded the beam file explicitly + ActualDir = filename:dirname(SuiteFile), + {[{ActualDir,Suite}|Found],NotFound}; + false -> + io:format(user, "Directory ~s is invalid~n", [Dir]), + Name = filename:join(Dir, atom_to_list(Suite)), + {Found,[{DS,[Name]}|NotFound]} + end end end, - lists:reverse(lists:foldl(Verify, [], TestSuites)). - - + {ActualFound,Missing} = lists:foldl(Verify, {[],[]}, TestSuites), + {lists:reverse(ActualFound),lists:reverse(Missing)}. + save_make_errors([]) -> []; save_make_errors(Errors) -> @@ -1096,7 +1351,7 @@ get_bad_suites([{{_TestDir,_Suite},Failed}|Errors], BadSuites) -> get_bad_suites([], BadSuites) -> BadSuites. - + %%%----------------------------------------------------------------- %%% @hidden @@ -1107,7 +1362,7 @@ step(TestDir, Suite, Case) -> %%%----------------------------------------------------------------- %%% @hidden %%% @equiv ct:step/4 -step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case), +step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case), Suite =/= all, Case =/= all -> do_run([{TestDir,Suite,Case}], [{step,Opts}]). @@ -1121,8 +1376,22 @@ suite_tuples([{TestDir,Suite,_} | Tests]) when is_atom(Suite) -> suite_tuples([]) -> []. -final_tests([{TestDir,Suites,_}|Tests], - Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) -> +final_tests(Tests, Skip, Bad) -> + + %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! + %%! io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]), + + {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad), + Skip2 = final_skip(Skip1, []), + + + %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! + %%! io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip2]), + + {Tests1,Skip2}. + +final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when + is_list(Suites), is_atom(hd(Suites)) -> % Separate = % fun(S,{DoSuite,Dont}) -> % case lists:keymember({TestDir,S},1,Bad) of @@ -1140,10 +1409,10 @@ final_tests([{TestDir,Suites,_}|Tests], Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites, S == S1, TD == TestDir], - Final1 = [{TestDir,S,all} || S <- Suites], - final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad); + Final1 = [{TestDir,S,all} || S <- Suites], + final_tests1(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad); -final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) -> +final_tests1([{TestDir,all,all}|Tests], Final, Skip, Bad) -> MissingSuites = case lists:keysearch({TestDir,all}, 1, Bad) of {value,{_,Failed}} -> @@ -1153,27 +1422,59 @@ final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) -> end, Missing = [{TestDir,S,"Make failed"} || S <- MissingSuites], Final1 = [{TestDir,all,all}|Final], - final_tests(Tests, Final1, Skip++Missing, Bad); + final_tests1(Tests, Final1, Skip++Missing, Bad); -final_tests([{TestDir,Suite,Cases}|Tests], - Final, Skip, Bad) when Cases==[]; Cases==all -> - final_tests([{TestDir,[Suite],all}|Tests], Final, Skip, Bad); +final_tests1([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) when + Cases==[]; Cases==all -> + final_tests1([{TestDir,[Suite],all}|Tests], Final, Skip, Bad); -final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) -> +final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when + is_list(GrsOrCs) -> case lists:keymember({TestDir,Suite}, 1, Bad) of - false -> - Do = {TestDir,Suite,Cases}, - final_tests(Tests, [Do|Final], Skip, Bad); true -> - Do = {TestDir,Suite,Cases}, - Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}], - final_tests(Tests, [Do|Final], Skip1, Bad) + Skip1 = Skip ++ [{TestDir,Suite,all,"Make failed"}], + final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad); + false -> + GrsOrCs1 = + lists:flatmap( + %% for now, only flat group defs are allowed as + %% start options and test spec terms + fun({all,all}) -> + ct_framework:make_all_conf(TestDir, + Suite, []); + ({skipped,Group,TCs}) -> + [ct_framework:make_conf(TestDir, Suite, + Group, [skipped], TCs)]; + ({Group,TCs}) -> + [ct_framework:make_conf(TestDir, Suite, + Group, [], TCs)]; + (TC) -> + [TC] + end, GrsOrCs), + Do = {TestDir,Suite,GrsOrCs1}, + final_tests1(Tests, [Do|Final], Skip, Bad) end; -final_tests([], Final, Skip, _Bad) -> +final_tests1([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. -continue([]) -> +final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) -> + SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all), + Skip = {TestDir,Suite,SkipConf,Reason}, + final_skip(Skips, [Skip|Final]); + +final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> + Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs), + Skip = {TestDir,Suite,Conf,Reason}, + final_skip(Skips, [Skip|Final]); + +final_skip([Skip|Skips], Final) -> + final_skip(Skips, [Skip|Final]); + +final_skip([], Final) -> + lists:reverse(Final). + +continue([]) -> true; continue(_MakeErrors) -> io:nl(), @@ -1214,7 +1515,7 @@ set_group_leader_same_as_shell() -> false end end, - case [P || P <- processes(), GS2or3(P), + case [P || P <- processes(), GS2or3(P), true == lists:keymember(shell,1,element(2,process_info(P,dictionary)))] of [GL|_] -> group_leader(GL, self()); @@ -1238,29 +1539,29 @@ check_and_add([{TestDir0,M,_} | Tests], Added) -> check_and_add([], _) -> ok. -do_run_test(Tests, Skip, Opt) -> +do_run_test(Tests, Skip, Opts) -> case check_and_add(Tests, []) of ok -> ct_util:set_testdata({stats,{0,0,{0,0}}}), ct_util:set_testdata({cover,undefined}), test_server_ctrl:start_link(local), - case lists:keysearch(cover_spec, 1, Opt) of - {value,{_,CovData={CovFile, - CovNodes, - _CovImport, - CovExport, - #cover{app = CovApp, - level = CovLevel, - excl_mods = CovExcl, - incl_mods = CovIncl, - cross = CovCross, - src = _CovSrc}}}} -> + case Opts#opts.coverspec of + CovData={CovFile, + CovNodes, + _CovImport, + CovExport, + #cover{app = CovApp, + level = CovLevel, + excl_mods = CovExcl, + incl_mods = CovIncl, + cross = CovCross, + src = _CovSrc}} -> ct_logs:log("COVER INFO","Using cover specification file: ~s~n" "App: ~w~n" "Cross cover: ~w~n" "Including ~w modules~n" "Excluding ~w modules", - [CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]), + [CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]), %% cover export file will be used for export and import %% between tests so make sure it doesn't exist initially @@ -1293,33 +1594,38 @@ do_run_test(Tests, Skip, Opt) -> true; _ -> false - end, + end, + %% let test_server expand the test tuples and count no of cases {Suites,NoOfCases} = count_test_cases(Tests, Skip), Suites1 = delete_dups(Suites), NoOfTests = length(Tests), NoOfSuites = length(Suites1), - ct_util:warn_duplicates(Suites1), + ct_util:warn_duplicates(Suites1), {ok,Cwd} = file:get_cwd(), io:format("~nCWD set to: ~p~n", [Cwd]), if NoOfCases == unknown -> - io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n", + io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n", [NoOfTests,NoOfSuites]), - ct_logs:log("TEST INFO","~w test(s), ~w suite(s)", + ct_logs:log("TEST INFO","~w test(s), ~w suite(s)", [NoOfTests,NoOfSuites]); true -> - io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n", + io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n", [NoOfTests,NoOfCases,NoOfSuites]), - ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)", + ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)", [NoOfTests,NoOfCases,NoOfSuites]) end, + + test_server_ctrl:multiply_timetraps(Opts#opts.multiply_timetraps), + test_server_ctrl:scale_timetraps(Opts#opts.scale_timetraps), + ct_event:notify(#event{name=start_info, node=node(), data={NoOfTests,NoOfSuites,NoOfCases}}), - CleanUp = add_jobs(Tests, Skip, Opt, []), + CleanUp = add_jobs(Tests, Skip, Opts, []), unlink(whereis(test_server_ctrl)), - catch test_server_ctrl:wait_finish(), - %% check if last testcase has left a "dead" trace window + catch test_server_ctrl:wait_finish(), + %% check if last testcase has left a "dead" trace window %% behind, and if so, kill it case ct_util:get_testdata(interpret) of {_What,kill,{TCPid,AttPid}} -> @@ -1327,8 +1633,8 @@ do_run_test(Tests, Skip, Opt) -> _ -> ok end, - lists:foreach(fun(Suite) -> - maybe_cleanup_interpret(Suite, Opt) + lists:foreach(fun(Suite) -> + maybe_cleanup_interpret(Suite, Opts#opts.step) end, CleanUp); Error -> Error @@ -1344,21 +1650,32 @@ count_test_cases(Tests, Skip) -> SendResult = fun(Me, Result) -> Me ! {no_of_cases,Result} end, TSPid = test_server_ctrl:start_get_totals(SendResult), Ref = erlang:monitor(process, TSPid), - add_jobs(Tests, Skip, [], []), - {Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref), - erlang:demonitor(Ref), - test_server_ctrl:stop_get_totals(), - {Suites,NoOfCases}. + add_jobs(Tests, Skip, #opts{}, []), + Counted = (catch count_test_cases1(length(Tests), 0, [], Ref)), + erlang:demonitor(Ref, [flush]), + case Counted of + {error,{test_server_died}} = Error -> + throw(Error); + {error,Reason} -> + unlink(whereis(test_server_ctrl)), + test_server_ctrl:stop(), + throw({user_error,Reason}); + Result -> + test_server_ctrl:stop_get_totals(), + Result + end. count_test_cases1(0, N, Suites, _) -> {lists:flatten(Suites), N}; count_test_cases1(Jobs, N, Suites, Ref) -> receive - {no_of_cases,{Ss,N1}} -> + {_,{error,_Reason} = Error} -> + throw(Error); + {no_of_cases,{Ss,N1}} -> count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref); - {'DOWN', Ref, _, _, _} -> - {[],0} - end. + {'DOWN', Ref, _, _, Info} -> + throw({error,{test_server_died,Info}}) + end. add_known(unknown, _) -> unknown; @@ -1367,72 +1684,109 @@ add_known(_, unknown) -> add_known(N, N1) -> N+N1. -add_jobs([{TestDir,all,_}|Tests], Skip, Opt, CleanUp) -> +add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) -> Name = get_name(TestDir), case catch test_server_ctrl:add_dir_with_skip(Name, TestDir, skiplist(TestDir,Skip)) of - {'EXIT',_} -> + {'EXIT',_} -> CleanUp; _ -> wait_for_idle(), - add_jobs(Tests, Skip, Opt, CleanUp) + add_jobs(Tests, Skip, Opts, CleanUp) end; -add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opt, CleanUp) when is_atom(Suite) -> - add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp); -add_jobs([{TestDir,Suites,all}|Tests], Skip, Opt, CleanUp) when is_list(Suites) -> +add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) -> + add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp); +add_jobs([{TestDir,Suites,all}|Tests], Skip, Opts, CleanUp) when is_list(Suites) -> Name = get_name(TestDir) ++ ".suites", case catch test_server_ctrl:add_module_with_skip(Name, Suites, skiplist(TestDir,Skip)) of - {'EXIT',_} -> + {'EXIT',_} -> CleanUp; _ -> wait_for_idle(), - add_jobs(Tests, Skip, Opt, CleanUp) + add_jobs(Tests, Skip, Opts, CleanUp) end; -add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp) -> - case maybe_interpret(Suite, all, Opt) of +add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> + case maybe_interpret(Suite, all, Opts) of ok -> Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite), case catch test_server_ctrl:add_module_with_skip(Name, [Suite], skiplist(TestDir,Skip)) of - {'EXIT',_} -> + {'EXIT',_} -> CleanUp; _ -> wait_for_idle(), - add_jobs(Tests, Skip, Opt, [Suite|CleanUp]) + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) end; Error -> Error end; -add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opt, CleanUp) when is_atom(Case) -> - add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp); -add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opt, CleanUp) when is_list(Cases) -> - case maybe_interpret(Suite, Cases, Opt) of + +%% group (= conf case in test_server) +add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when + element(1, hd(Confs)) == conf -> + Group = fun(Conf) -> proplists:get_value(name, element(2, Conf)) end, + TestCases = fun(Conf) -> element(4, Conf) end, + TCTestName = fun(all) -> ""; + ([C]) when is_atom(C) -> "." ++ atom_to_list(C); + (Cs) when is_list(Cs) -> ".cases" + end, + GrTestName = + case Confs of + [Conf] -> + "." ++ atom_to_list(Group(Conf)) ++ TCTestName(TestCases(Conf)); + _ -> + ".groups" + end, + TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName, + case maybe_interpret(Suite, init_per_group, Opts) of + ok -> + case catch test_server_ctrl:add_conf_with_skip(TestName, Suite, Confs, + skiplist(TestDir,Skip)) of + {'EXIT',_} -> + CleanUp; + _ -> + wait_for_idle(), + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) + end; + Error -> + Error + end; + +%% test case +add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -> + add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp); + +add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opts, CleanUp) when is_list(Cases) -> + Cases1 = lists:map(fun({GroupName,_}) when is_atom(GroupName) -> GroupName; + (Case) -> Case + end, Cases), + case maybe_interpret(Suite, Cases1, Opts) of ok -> Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases", - case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases, + case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases1, skiplist(TestDir,Skip)) of - {'EXIT',_} -> + {'EXIT',_} -> CleanUp; _ -> wait_for_idle(), - add_jobs(Tests, Skip, Opt, [Suite|CleanUp]) + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) end; Error -> Error end; -add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp) when is_atom(Case) -> - case maybe_interpret(Suite, Case, Opt) of +add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -> + case maybe_interpret(Suite, Case, Opts) of ok -> - Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++ + Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++ atom_to_list(Case), case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case, skiplist(TestDir,Skip)) of - {'EXIT',_} -> + {'EXIT',_} -> CleanUp; _ -> wait_for_idle(), - add_jobs(Tests, Skip, Opt, [Suite|CleanUp]) + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) end; Error -> Error @@ -1453,7 +1807,7 @@ wait_for_idle() -> idle -> ok; {'DOWN', Ref, _, _, _} -> error end, - erlang:demonitor(Ref), + erlang:demonitor(Ref, [flush]), ct_util:update_last_run_index(), Result end. @@ -1482,7 +1836,7 @@ get_name(Dir) -> end, Base = filename:basename(TestDir), case filename:basename(filename:dirname(TestDir)) of - "" -> + "" -> Base; TopDir -> TopDir ++ "." ++ Base @@ -1513,15 +1867,15 @@ run_make(Targets, TestDir0, Mod, UserInclude) -> {i,CtInclude}, {i,XmerlInclude}, debug_info], - Result = + Result = if Mod == all ; Targets == helpmods -> case (catch ct_make:all([noexec|ErlFlags])) of - {'EXIT',_} = Failure -> + {'EXIT',_} = Failure -> Failure; MakeInfo -> FileTest = fun(F, suites) -> is_suite(F); - (F, helpmods) -> not is_suite(F); - (_, _) -> true end, + (F, helpmods) -> not is_suite(F) + end, Files = lists:flatmap(fun({F,out_of_date}) -> case FileTest(F, Targets) of true -> [F]; @@ -1535,7 +1889,7 @@ run_make(Targets, TestDir0, Mod, UserInclude) -> true -> (catch ct_make:files([Mod], [load|ErlFlags])) end, - + ok = file:set_cwd(Cwd), %% send finished_make notification ct_event:notify(#event{name=finished_make, @@ -1549,7 +1903,7 @@ run_make(Targets, TestDir0, Mod, UserInclude) -> {error,{make_crashed,TestDir,Reason}}; {error,ModInfo} -> io:format("{error,make_failed}\n", []), - Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo, + Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo, R == error], {error,{make_failed,Bad}} end; @@ -1561,8 +1915,8 @@ run_make(Targets, TestDir0, Mod, UserInclude) -> get_dir(App, Dir) -> filename:join(code:lib_dir(App), Dir). -maybe_interpret(Suite, Cases, [{step,StepOpts}]) -> - %% if other suite has run before this one, check if last testcase +maybe_interpret(Suite, Cases, #opts{step = StepOpts}) when StepOpts =/= undefined -> + %% if other suite has run before this one, check if last testcase %% has left a "dead" trace window behind, and if so, kill it case ct_util:get_testdata(interpret) of {_What,kill,{TCPid,AttPid}} -> @@ -1605,7 +1959,7 @@ maybe_interpret2(Suite, Cases, StepOpts) -> WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of true -> no_kill; false -> kill - end, + end, ct_util:set_testdata({interpret,{{Suite,Cases},WinOp, {undefined,undefined}}}), ok. @@ -1621,37 +1975,44 @@ set_break_on_config(Suite, StepOpts) -> ok end. -maybe_cleanup_interpret(Suite, [{step,_}]) -> - i:iq(Suite); -maybe_cleanup_interpret(_, _) -> - ok. +maybe_cleanup_interpret(_, undefined) -> + ok; +maybe_cleanup_interpret(Suite, _) -> + i:iq(Suite). + +log_ts_names([]) -> + ok; +log_ts_names(Specs) -> + List = lists:map(fun(Name) -> + Name ++ " " + end, Specs), + ct_logs:log("Test Specification file(s)", "~s", + [lists:flatten(List)]). -log_ts_names(Args) -> - case lists:keysearch(spec, 1, Args) of - {value,{_,Specs}} -> - List = lists:map(fun(Name) -> - Name ++ " " - end, Specs), - ct_logs:log("Test Specification file(s)", "~s", - [lists:flatten(List)]); - _ -> - ok - end. - merge_arguments(Args) -> merge_arguments(Args, []). merge_arguments([LogDir={logdir,_}|Args], Merged) -> merge_arguments(Args, handle_arg(replace, LogDir, Merged)); + merge_arguments([CoverFile={cover,_}|Args], Merged) -> merge_arguments(Args, handle_arg(replace, CoverFile, Merged)); -merge_arguments([Arg={_,_}|Args], Merged) -> + +merge_arguments([{'case',TC}|Args], Merged) -> + merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged)); + +merge_arguments([Arg|Args], Merged) -> merge_arguments(Args, handle_arg(merge, Arg, Merged)); + merge_arguments([], Merged) -> Merged. handle_arg(replace, {Key,Elems}, [{Key,_}|Merged]) -> [{Key,Elems}|Merged]; +handle_arg(merge, {event_handler_init,Elems}, [{event_handler_init,PrevElems}|Merged]) -> + [{event_handler_init,PrevElems++["add"|Elems]}|Merged]; +handle_arg(merge, {userconfig,Elems}, [{userconfig,PrevElems}|Merged]) -> + [{userconfig,PrevElems++["add"|Elems]}|Merged]; handle_arg(merge, {Key,Elems}, [{Key,PrevElems}|Merged]) -> [{Key,PrevElems++Elems}|Merged]; handle_arg(Op, Arg, [Other|Merged]) -> @@ -1659,6 +2020,164 @@ handle_arg(Op, Arg, [Other|Merged]) -> handle_arg(_,Arg,[]) -> [Arg]. +get_start_opt(Key, IfExists, Args) -> + get_start_opt(Key, IfExists, undefined, Args). + +get_start_opt(Key, IfExists, IfNotExists, Args) -> + case lists:keysearch(Key, 1, Args) of + {value,{Key,Val}} when is_function(IfExists) -> + IfExists(Val); + {value,{Key,Val}} when IfExists == value -> + Val; + {value,{Key,_Val}} -> + IfExists; + _ when is_function(IfNotExists) -> + IfNotExists(); + _ -> + IfNotExists + end. + +event_handler_args2opts(Args) -> + case proplists:get_value(event_handler, Args) of + undefined -> + event_handler_args2opts([], Args); + EHs -> + event_handler_args2opts([{list_to_atom(EH),[]} || EH <- EHs], Args) + end. +event_handler_args2opts(Default, Args) -> + case proplists:get_value(event_handler_init, Args) of + undefined -> + Default; + EHs -> + event_handler_init_args2opts(EHs) + end. +event_handler_init_args2opts([EH, Arg, "and" | EHs]) -> + [{list_to_atom(EH),lists:flatten(io_lib:format("~s",[Arg]))} | + event_handler_init_args2opts(EHs)]; +event_handler_init_args2opts([EH, Arg]) -> + [{list_to_atom(EH),lists:flatten(io_lib:format("~s",[Arg]))}]; +event_handler_init_args2opts([]) -> + []. + +%% This function reads pa and pz arguments, converts dirs from relative +%% to absolute, and re-inserts them in the code path. The order of the +%% dirs in the code path remain the same. Note however that since this +%% function is only used for arguments "pre run_test erl_args", the order +%% relative dirs "post run_test erl_args" is not kept! +rel_to_abs(CtArgs) -> + {PA,PZ} = get_pa_pz(CtArgs, [], []), + io:format(user, "~n", []), + [begin + code:del_path(filename:basename(D)), + Abs = filename:absname(D), + code:add_pathz(Abs), + if D /= Abs -> + io:format(user, "Converting ~p to ~p and re-inserting " + "with add_pathz/1~n", + [D, Abs]); + true -> + ok + end + end || D <- PZ], + [begin + code:del_path(filename:basename(D)), + Abs = filename:absname(D), + code:add_patha(Abs), + if D /= Abs -> + io:format(user, "Converting ~p to ~p and re-inserting " + "with add_patha/1~n", + [D, Abs]); + true ->ok + end + end || D <- PA], + io:format(user, "~n", []). + +get_pa_pz([{pa,Dirs} | Args], PA, PZ) -> + get_pa_pz(Args, PA ++ Dirs, PZ); +get_pa_pz([{pz,Dirs} | Args], PA, PZ) -> + get_pa_pz(Args, PA, PZ ++ Dirs); +get_pa_pz([_ | Args], PA, PZ) -> + get_pa_pz(Args, PA, PZ); +get_pa_pz([], PA, PZ) -> + {PA,PZ}. + +%% This function translates ct:run_test/1 start options +%% to run_test start arguments (on the init arguments format) - +%% this is useful mainly for testing the ct_run start functions. +opts2args(EnvStartOpts) -> + lists:flatmap(fun({config,CfgFiles}) -> + [{ct_config,[CfgFiles]}]; + ({userconfig,{CBM,CfgStr=[X|_]}}) when is_integer(X) -> + [{userconfig,[atom_to_list(CBM),CfgStr]}]; + ({userconfig,{CBM,CfgStrs}}) when is_list(CfgStrs) -> + [{userconfig,[atom_to_list(CBM) | CfgStrs]}]; + ({userconfig,UserCfg}) when is_list(UserCfg) -> + Strs = + lists:map(fun({CBM,CfgStr=[X|_]}) when is_integer(X) -> + [atom_to_list(CBM),CfgStr,"and"]; + ({CBM,CfgStrs}) when is_list(CfgStrs) -> + [atom_to_list(CBM) | CfgStrs] ++ ["and"] + end, UserCfg), + [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), + [{userconfig,lists:reverse(StrsR)}]; + ({testcase,Case}) when is_atom(Case) -> + [{'case',[atom_to_list(Case)]}]; + ({testcase,Cases}) -> + [{'case',[atom_to_list(C) || C <- Cases]}]; + ({'case',Cases}) -> + [{'case',[atom_to_list(C) || C <- Cases]}]; + ({allow_user_terms,true}) -> + [{allow_user_terms,[]}]; + ({allow_user_terms,false}) -> + []; + ({auto_compile,false}) -> + [{no_auto_compile,[]}]; + ({auto_compile,true}) -> + []; + ({scale_timetraps,true}) -> + [{scale_timetraps,[]}]; + ({scale_timetraps,false}) -> + []; + ({force_stop,true}) -> + [{force_stop,[]}]; + ({force_stop,false}) -> + []; + ({decrypt,{key,Key}}) -> + [{ct_decrypt_key,[Key]}]; + ({decrypt,{file,File}}) -> + [{ct_decrypt_file,[File]}]; + ({basic_html,true}) -> + ({basic_html,[]}); + ({basic_html,false}) -> + []; + ({event_handler,EH}) when is_atom(EH) -> + [{event_handler,[atom_to_list(EH)]}]; + ({event_handler,EHs}) when is_list(EHs) -> + [{event_handler,[atom_to_list(EH) || EH <- EHs]}]; + ({event_handler,{EH,Arg}}) when is_atom(EH) -> + ArgStr = lists:flatten(io_lib:format("~p", [Arg])), + [{event_handler_init,[atom_to_list(EH),ArgStr]}]; + ({event_handler,{EHs,Arg}}) when is_list(EHs) -> + ArgStr = lists:flatten(io_lib:format("~p", [Arg])), + Strs = lists:map(fun(EH) -> + [atom_to_list(EH),ArgStr,"and"] + end, EHs), + [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), + [{event_handler_init,lists:reverse(StrsR)}]; + ({Opt,As=[A|_]}) when is_atom(A) -> + [{Opt,[atom_to_list(Atom) || Atom <- As]}]; + ({Opt,Strs=[S|_]}) when is_list(S) -> + [{Opt,Strs}]; + ({Opt,A}) when is_atom(A) -> + [{Opt,[atom_to_list(A)]}]; + ({Opt,I}) when is_integer(I) -> + [{Opt,[integer_to_list(I)]}]; + ({Opt,S}) when is_list(S) -> + [{Opt,[S]}]; + (Opt) -> + Opt + end, EnvStartOpts). + locate_test_dir(Dir, Suite) -> TestDir = case ct_util:is_test_dir(Dir) of true -> Dir; @@ -1723,18 +2242,18 @@ start_trace(Args) -> case file:consult(TraceSpec) of {ok,Terms} -> case catch do_trace(Terms) of - ok -> + ok -> true; {_,Error} -> io:format("Warning! Tracing not started. Reason: ~p~n~n", [Error]), false - end; + end; {_,Error} -> io:format("Warning! Tracing not started. Reason: ~p~n~n", [Error]), false - end; + end; false -> false end. @@ -1746,61 +2265,22 @@ do_trace(Terms) -> case dbg:tpl(M,[{'_',[],[{return_trace}]}]) of {error,What} -> exit({error,{tracing_failed,What}}); _ -> ok - end; + end; ({f,M,F}) -> case dbg:tpl(M,F,[{'_',[],[{return_trace}]}]) of {error,What} -> exit({error,{tracing_failed,What}}); _ -> ok - end; + end; (Huh) -> exit({error,{unrecognized_trace_term,Huh}}) end, Terms), ok. - + stop_trace(true) -> dbg:stop_clear(); stop_trace(false) -> ok. -preload() -> - io:format("~nLoading Common Test and Test Server modules...~n~n"), - preload_mod([ct_logs, - ct_make, - ct_telnet, - ct, - ct_master, - ct_testspec, - ct_cover, - ct_master_event, - ct_util, - ct_event, - ct_master_logs, - ct_framework, - teln, - ct_ftp, - ct_rpc, - unix_telnet, - ct_gen_conn, - ct_line, - ct_snmp, - test_server_sup, - test_server, - test_server_ctrl, - test_server_h, - test_server_line, - test_server_node]). - -preload_mod([M|Ms]) -> - case code:is_loaded(M) of - false -> - {module,M} = code:load_file(M), - preload_mod(Ms); - _ -> - ok - end; -preload_mod([]) -> - ok. - ensure_atom(Atom) when is_atom(Atom) -> Atom; ensure_atom(String) when is_list(String), is_integer(hd(String)) -> @@ -1809,4 +2289,3 @@ ensure_atom(List) when is_list(List) -> [ensure_atom(Item) || Item <- List]; ensure_atom(Other) -> Other. - diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl new file mode 100644 index 0000000000..d2a491e079 --- /dev/null +++ b/lib/common_test/src/ct_slave.erl @@ -0,0 +1,439 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% + +%%% @doc Common Test Framework functions for starting and stopping nodes for +%%% Large Scale Testing. +%%% +%%% <p>This module exports functions which are used by the Common Test Master +%%% to start and stop "slave" nodes. It is the default callback module for the +%%% <code>{init, node_start}</code> term of the Test Specification.</p> + +%%---------------------------------------------------------------------- +%% File : ct_slave.erl +%% Description : CT module for starting nodes for large-scale testing. +%% +%% Created : 7 April 2010 +%%---------------------------------------------------------------------- +-module(ct_slave). + +-export([start/1, start/2, start/3, stop/1, stop/2]). + +-export([slave_started/2, slave_ready/2, monitor_master/1]). + +-record(options, {username, password, boot_timeout, init_timeout, + startup_timeout, startup_functions, monitor_master, + kill_if_fail, erl_flags}). + +%%%----------------------------------------------------------------- +%%% @spec start(Node) -> Result +%%% Node = atom() +%%% Result = {ok, NodeName} | +%%% {error, already_started, NodeName} | +%%% {error, started_not_connected, NodeName} | +%%% {error, boot_timeout, NodeName} | +%%% {error, init_timeout, NodeName} | +%%% {error, startup_timeout, NodeName} | +%%% {error, not_alive, NodeName} +%%% NodeName = atom() +%%% @doc Starts an Erlang node with name <code>Node</code> on the local host. +%%% @see start/3 +start(Node)-> + start(gethostname(), Node). + +%%%----------------------------------------------------------------- +%%% @spec start(Host, Node) -> Result +%%% Node = atom() +%%% Host = atom() +%%% Result = {ok, NodeName} | +%%% {error, already_started, NodeName} | +%%% {error, started_not_connected, NodeName} | +%%% {error, boot_timeout, NodeName} | +%%% {error, init_timeout, NodeName} | +%%% {error, startup_timeout, NodeName} | +%%% {error, not_alive, NodeName} +%%% NodeName = atom() +%%% @doc Starts an Erlang node with name <code>Node</code> on host +%%% <code>Host</code> with the default options. +%%% @see start/3 +start(Host, Node)-> + start(Host, Node, []). + +%%%----------------------------------------------------------------- +%%% @spec start(Host, Node, Opts) -> Result +%%% Node = atom() +%%% Host = atom() +%%% Opts = [OptTuples] +%%% OptTuples = {username, Username} | +%%% {password, Password} | +%%% {boot_timeout, BootTimeout} | {init_timeout, InitTimeout} | +%%% {startup_timeout, StartupTimeout} | +%%% {startup_functions, StartupFunctions} | +%%% {monitor_master, Monitor} | +%%% {kill_if_fail, KillIfFail} | +%%% {erl_flags, ErlangFlags} +%%% Username = string() +%%% Password = string() +%%% BootTimeout = integer() +%%% InitTimeout = integer() +%%% StartupTimeout = integer() +%%% StartupFunctions = [StartupFunctionSpec] +%%% StartupFunctionSpec = {Module, Function, Arguments} +%%% Module = atom() +%%% Function = atom() +%%% Arguments = [term] +%%% Monitor = bool() +%%% KillIfFail = bool() +%%% ErlangFlags = string() +%%% Result = {ok, NodeName} | {error, already_started, NodeName} | +%%% {error, started_not_connected, NodeName} | +%%% {error, boot_timeout, NodeName} | +%%% {error, init_timeout, NodeName} | +%%% {error, startup_timeout, NodeName} | +%%% {error, not_alive, NodeName} +%%% NodeName = atom() +%%% @doc Starts an Erlang node with name <code>Node</code> on host +%%% <code>Host</code> as specified by the combination of options in +%%% <code>Opts</code>. +%%% +%%% <p>Options <code>Username</code> and <code>Password</code> will be used +%%% to log in onto the remote host <code>Host</code>. +%%% Username, if omitted, defaults to the current user name, +%%% and password is empty by default.</p> +%%% +%%% <p>A list of functions specified in the <code>Startup</code> option will be +%%% executed after startup of the node. Note that all used modules should be +%%% present in the code path on the <code>Host</code>.</p> +%%% +%%% <p>The timeouts are applied as follows: +%%% <list> +%%% <item> +%%% <code>BootTimeout</code> - time to start the Erlang node, in seconds. +%%% Defaults to 3 seconds. If node does not become pingable within this time, +%%% the result <code>{error, boot_timeout, NodeName}</code> is returned; +%%% </item> +%%% <item> +%%% <code>InitTimeout</code> - time to wait for the node until it calls the +%%% internal callback function informing master about successfull startup. +%%% Defaults to one second. +%%% In case of timed out message the result +%%% <code>{error, init_timeout, NodeName}</code> is returned; +%%% </item> +%%% <item> +%%% <code>StartupTimeout</code> - time to wait intil the node finishes to run +%%% the <code>StartupFunctions</code>. Defaults to one second. +%%% If this timeout occurs, the result +%%% <code>{error, startup_timeout, NodeName}</code> is returned. +%%% </item> +%%% </list></p> +%%% +%%% <p>Option <code>monitor_master</code> specifies, if the slave node should be +%%% stopped in case of master node stop. Defaults to false.</p> +%%% +%%% <p>Option <code>kill_if_fail</code> specifies, if the slave node should be +%%% killed in case of a timeout during initialization or startup. +%%% Defaults to true. Note that node also may be still alive it the boot +%%% timeout occurred, but it will not be killed in this case.</p> +%%% +%%% <p>Option <code>erlang_flags</code> specifies, which flags will be added +%%% to the parameters of the <code>erl</code> executable.</p> +%%% +%%% <p>Special return values are: +%%% <list> +%%% <item><code>{error, already_started, NodeName}</code> - if the node with +%%% the given name is already started on a given host;</item> +%%% <item><code>{error, started_not_connected, NodeName}</code> - if node is +%%% started, but not connected to the master node.</item> +%%% <item><code>{error, not_alive, NodeName}</code> - if node on which the +%%% <code>ct_slave:start/3</code> is called, is not alive. Note that +%%% <code>NodeName</code> is the name of current node in this case.</item> +%%% </list></p> +%%% +start(Host, Node, Options)-> + ENode = enodename(Host, Node), + case erlang:is_alive() of + false-> + {error, not_alive, node()}; + true-> + case is_started(ENode) of + false-> + OptionsRec = fetch_options(Options), + do_start(Host, Node, OptionsRec); + {true, not_connected}-> + {error, started_not_connected, ENode}; + {true, connected}-> + {error, already_started, ENode} + end + end. + +%%% @spec stop(Node) -> Result +%%% Node = atom() +%%% Result = {ok, NodeName} | +%%% {error, not_started, NodeName} | +%%% {error, not_connected, NodeName} | +%%% {error, stop_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Stops the running Erlang node with name <code>Node</code> on +%%% the localhost. +stop(Node)-> + stop(gethostname(), Node). + +%%% @spec stop(Host, Node) -> Result +%%% Host = atom() +%%% Node = atom() +%%% Result = {ok, NodeName} | +%%% {error, not_started, NodeName} | +%%% {error, not_connected, NodeName} | +%%% {error, stop_timeout, NodeName} +%%% NodeName = atom() +%%% @doc Stops the running Erlang node with name <code>Node</code> on +%%% host <code>Host</code>. +stop(Host, Node)-> + ENode = enodename(Host, Node), + case is_started(ENode) of + {true, connected}-> + do_stop(ENode); + {true, not_connected}-> + {error, not_connected, ENode}; + false-> + {error, not_started, ENode} + end. + +%%% fetch an option value from the tagged tuple list with default +get_option_value(Key, OptionList, Default)-> + case lists:keyfind(Key, 1, OptionList) of + false-> + Default; + {Key, Value}-> + Value + end. + +%%% convert option list to the option record, fill all defaults +fetch_options(Options)-> + UserName = get_option_value(username, Options, []), + Password = get_option_value(password, Options, []), + BootTimeout = get_option_value(boot_timeout, Options, 3), + InitTimeout = get_option_value(init_timeout, Options, 1), + StartupTimeout = get_option_value(startup_timeout, Options, 1), + StartupFunctions = get_option_value(startup_functions, Options, []), + Monitor = get_option_value(monitor_master, Options, false), + KillIfFail = get_option_value(kill_if_fail, Options, true), + ErlFlags = get_option_value(erl_flags, Options, []), + #options{username=UserName, password=Password, + boot_timeout=BootTimeout, init_timeout=InitTimeout, + startup_timeout=StartupTimeout, startup_functions=StartupFunctions, + monitor_master=Monitor, kill_if_fail=KillIfFail, erl_flags=ErlFlags}. + +% send a message when slave node is started +% @hidden +slave_started(ENode, MasterPid)-> + MasterPid ! {node_started, ENode}, + ok. + +% send a message when slave node has finished startup +% @hidden +slave_ready(ENode, MasterPid)-> + MasterPid ! {node_ready, ENode}, + ok. + +% start monitoring of the master node +% @hidden +monitor_master(MasterNode)-> + spawn(fun()->monitor_master_int(MasterNode) end). + +% code of the masterdeath-waiter process +monitor_master_int(MasterNode)-> + erlang:monitor_node(MasterNode, true), + receive + {nodedown, MasterNode}-> + init:stop() + end. + +% check if node is listed in the nodes() +is_connected(ENode)-> + [N||N<-nodes(), N==ENode] == [ENode]. + +% check if node is alive (ping and disconnect if pingable) +is_started(ENode)-> + case is_connected(ENode) of + true-> + {true, connected}; + false-> + case net_adm:ping(ENode) of + pang-> + false; + pong-> + erlang:disconnect_node(ENode), + {true, not_connected} + end + end. + +% make a Erlang node name from name and hostname +enodename(Host, Node)-> + list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)). + +% performs actual start of the "slave" node +do_start(Host, Node, Options)-> + ENode = enodename(Host, Node), + Functions = + lists:concat([[{ct_slave, slave_started, [ENode, self()]}], + Options#options.startup_functions, + [{ct_slave, slave_ready, [ENode, self()]}]]), + Functions2 = if + Options#options.monitor_master-> + [{ct_slave, monitor_master, [node()]}|Functions]; + true-> + Functions + end, + MasterHost = gethostname(), + if + MasterHost == Host -> + spawn_local_node(Node, Options); + true-> + spawn_remote_node(Host, Node, Options) + end, + BootTimeout = Options#options.boot_timeout, + InitTimeout = Options#options.init_timeout, + StartupTimeout = Options#options.startup_timeout, + Result = case wait_for_node_alive(ENode, BootTimeout) of + pong-> + call_functions(ENode, Functions2), + receive + {node_started, ENode}-> + receive + {node_ready, ENode}-> + {ok, ENode} + after StartupTimeout*1000-> + {error, startup_timeout, ENode} + end + after InitTimeout*1000 -> + {error, init_timeout, ENode} + end; + pang-> + {error, boot_timeout, ENode} + end, + case Result of + {ok, ENode}-> + ok; + {error, Timeout, ENode} + when ((Timeout==init_timeout) or (Timeout==startup_timeout)) and + Options#options.kill_if_fail-> + do_stop(ENode); + _-> ok + end, + Result. + +% are we using fully qualified hostnames +long_or_short()-> + case net_kernel:longnames() of + true-> + " -name "; + false-> + " -sname " + end. + +% get the localhost's name, depending on the using name policy +gethostname()-> + Hostname = case net_kernel:longnames() of + true-> + net_adm:localhost(); + _-> + {ok, Name}=inet:gethostname(), + Name + end, + list_to_atom(Hostname). + +% get cmd for starting Erlang +get_cmd(Node, Flags)-> + Cookie = erlang:get_cookie(), + "erl -detached -noinput -setcookie "++ atom_to_list(Cookie) ++ + long_or_short() ++ atom_to_list(Node) ++ " " ++ Flags. + +% spawn node locally +spawn_local_node(Node, Options)-> + ErlFlags = Options#options.erl_flags, + Cmd = get_cmd(Node, ErlFlags), + open_port({spawn, Cmd}, [stream]). + +% start crypto and ssh if not yet started +check_for_ssh_running()-> + case application:get_application(crypto) of + undefined-> + application:start(crypto), + case application:get_application(ssh) of + undefined-> + application:start(ssh); + {ok, ssh}-> + ok + end; + {ok, crypto}-> + ok + end. + +% spawn node remotely +spawn_remote_node(Host, Node, Options)-> + Username = Options#options.username, + Password = Options#options.password, + ErlFlags = Options#options.erl_flags, + SSHOptions = case {Username, Password} of + {[], []}-> + []; + {_, []}-> + [{user, Username}]; + {_, _}-> + [{user, Username}, {password, Password}] + end ++ [{silently_accept_hosts, true}], + check_for_ssh_running(), + {ok, SSHConnRef} = ssh:connect(atom_to_list(Host), 22, SSHOptions), + {ok, SSHChannelId} = ssh_connection:session_channel(SSHConnRef, infinity), + ssh_connection:exec(SSHConnRef, SSHChannelId, get_cmd(Node, ErlFlags), infinity). + +% call functions on a remote Erlang node +call_functions(_Node, [])-> + ok; +call_functions(Node, [{M, F, A}|Functions])-> + rpc:call(Node, M, F, A), + call_functions(Node, Functions). + +% wait N seconds until node is pingable +wait_for_node_alive(_Node, 0)-> + pang; +wait_for_node_alive(Node, N)-> + timer:sleep(1000), + case net_adm:ping(Node) of + pong-> + pong; + pang-> + wait_for_node_alive(Node, N-1) + end. + +% call init:stop on a remote node +do_stop(ENode)-> + spawn(ENode, init, stop, []), + wait_for_node_dead(ENode, 5). + +% wait N seconds until node is disconnected +wait_for_node_dead(Node, 0)-> + {error, stop_timeout, Node}; +wait_for_node_dead(Node, N)-> + timer:sleep(1000), + case lists:member(Node, nodes()) of + true-> + wait_for_node_dead(Node, N-1); + false-> + {ok, Node} + end. diff --git a/lib/common_test/src/ct_snmp.erl b/lib/common_test/src/ct_snmp.erl index 7ff88ad7d3..8fe63e8ed1 100644 --- a/lib/common_test/src/ct_snmp.erl +++ b/lib/common_test/src/ct_snmp.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2004-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -332,7 +332,7 @@ set_info(Config) -> register_users(MgrAgentConfName, Users) -> {snmp, SnmpVals} = ct:get_config(MgrAgentConfName), NewSnmpVals = lists:keyreplace(users, 1, SnmpVals, {users, Users}), - ct_util:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), + ct_config:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), setup_users(Users). %%% @spec register_agents(MgrAgentConfName, ManagedAgents) -> ok | {error, Reason} @@ -347,7 +347,7 @@ register_agents(MgrAgentConfName, ManagedAgents) -> {snmp, SnmpVals} = ct:get_config(MgrAgentConfName), NewSnmpVals = lists:keyreplace(managed_agents, 1, SnmpVals, {managed_agents, ManagedAgents}), - ct_util:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), + ct_config:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), setup_managed_agents(ManagedAgents). %%% @spec register_usm_users(MgrAgentConfName, UsmUsers) -> ok | {error, Reason} @@ -361,7 +361,7 @@ register_agents(MgrAgentConfName, ManagedAgents) -> register_usm_users(MgrAgentConfName, UsmUsers) -> {snmp, SnmpVals} = ct:get_config(MgrAgentConfName), NewSnmpVals = lists:keyreplace(users, 1, SnmpVals, {usm_users, UsmUsers}), - ct_util:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), + ct_config:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), EngineID = ct:get_config({MgrAgentConfName, engine_id}, ?ENGINE_ID), setup_usm_users(UsmUsers, EngineID). @@ -376,7 +376,7 @@ unregister_users(MgrAgentConfName) -> ct:get_config({MgrAgentConfName, users})), {snmp, SnmpVals} = ct:get_config(MgrAgentConfName), NewSnmpVals = lists:keyreplace(users, 1, SnmpVals, {users, []}), - ct_util:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), + ct_config:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), takedown_users(Users). %%% @spec unregister_agents(MgrAgentConfName) -> ok | {error, Reason} @@ -393,7 +393,7 @@ unregister_agents(MgrAgentConfName) -> {snmp, SnmpVals} = ct:get_config(MgrAgentConfName), NewSnmpVals = lists:keyreplace(managed_agents, 1, SnmpVals, {managed_agents, []}), - ct_util:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), + ct_config:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), takedown_managed_agents(ManagedAgents). @@ -409,7 +409,7 @@ update_usm_users(MgrAgentConfName, UsmUsers) -> {snmp, SnmpVals} = ct:get_config(MgrAgentConfName), NewSnmpVals = lists:keyreplace(usm_users, 1, SnmpVals, {usm_users, UsmUsers}), - ct_util:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), + ct_config:update_config(MgrAgentConfName, {snmp, NewSnmpVals}), EngineID = ct:get_config({MgrAgentConfName, engine_id}, ?ENGINE_ID), do_update_usm_users(UsmUsers, EngineID). diff --git a/lib/common_test/src/ct_ssh.erl b/lib/common_test/src/ct_ssh.erl index f2b25b1fcd..aebb28bc42 100644 --- a/lib/common_test/src/ct_ssh.erl +++ b/lib/common_test/src/ct_ssh.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -961,24 +961,25 @@ init(KeyOrName, {ConnType,Addr,Port}, AllOpts) -> ssh -> ssh:connect(Addr, Port, FinalOptions); sftp -> - ssh_sftp:connect(Addr, Port, FinalOptions) + ssh_sftp:start_channel(Addr, Port, FinalOptions) end, case Result of - {ok,SSHRef} -> + Error = {error,_} -> + Error; + Ok -> + SSHRef = element(2, Ok), log(heading(init,KeyOrName), "Opened ~w connection:\nHost: ~p (~p)\nUser: ~p\nPassword: ~p\n", [ConnType,Addr,Port,User,lists:duplicate(length(Password),$*)]), {ok,SSHRef,#state{ssh_ref=SSHRef, conn_type=ConnType, - target=KeyOrName}}; - Error -> - Error + target=KeyOrName}} end. %% @hidden handle_msg(sftp_connect, State) -> #state{ssh_ref=SSHRef, target=Target} = State, log(heading(sftp_connect,Target), "SSH Ref: ~p", [SSHRef]), - {ssh_sftp:connect(SSHRef),State}; + {ssh_sftp:start_channel(SSHRef),State}; handle_msg({session_open,TO}, State) -> #state{ssh_ref=SSHRef, target=Target} = State, @@ -1202,7 +1203,7 @@ terminate(SSHRef, State) -> sftp -> log(heading(disconnect_sftp,State#state.target), "SFTP Ref: ~p",[SSHRef]), - ssh_sftp:stop(SSHRef) + ssh_sftp:stop_channel(SSHRef) end. @@ -1213,7 +1214,6 @@ terminate(SSHRef, State) -> %%% do_recv_response(SSH, Chn, Data, End, Timeout) -> receive - {ssh_cm, SSH, {open,Chn,RemoteChn,{session}}} -> debug("RECVD open"), {ok,{open,Chn,RemoteChn,{session}}}; diff --git a/lib/common_test/src/ct_telnet_client.erl b/lib/common_test/src/ct_telnet_client.erl index 1a12c5e343..d703b39ac5 100644 --- a/lib/common_test/src/ct_telnet_client.erl +++ b/lib/common_test/src/ct_telnet_client.erl @@ -35,8 +35,6 @@ -export([open/1, open/2, open/3, open/4, close/1]). -export([send_data/2, get_data/1]). --define(DBG, false). - -define(TELNET_PORT, 23). -define(OPEN_TIMEOUT,10000). -define(IDLE_TIMEOUT,10000). @@ -287,35 +285,38 @@ get_subcmd([?SE | Rest], Acc) -> get_subcmd([Opt | Rest], Acc) -> get_subcmd(Rest, [Opt | Acc]). - +-ifdef(debug). dbg(_Str,_Args) -> - if ?DBG -> io:format(_Str,_Args); - true -> ok + io:format(_Str,_Args). + +cmd_dbg(_Cmd) -> + case _Cmd of + [?IAC|Cmd1] -> + cmd_dbg(Cmd1); + [Ctrl|Opts] -> + CtrlStr = + case Ctrl of + ?DO -> "DO"; + ?DONT -> "DONT"; + ?WILL -> "WILL"; + ?WONT -> "WONT"; + ?NOP -> "NOP"; + _ -> "CMD" + end, + Opts1 = + case Opts of + [Opt] -> Opt; + _ -> Opts + end, + io:format("~s(~w): ~w\n", [CtrlStr,Ctrl,Opts1]); + Any -> + io:format("Unexpected in cmd_dbg:~n~w~n",[Any]) end. +-else. +dbg(_Str,_Args) -> + ok. + cmd_dbg(_Cmd) -> - if ?DBG -> - case _Cmd of - [?IAC|Cmd1] -> - cmd_dbg(Cmd1); - [Ctrl|Opts] -> - CtrlStr = - case Ctrl of - ?DO -> "DO"; - ?DONT -> "DONT"; - ?WILL -> "WILL"; - ?WONT -> "WONT"; - ?NOP -> "NOP"; - _ -> "CMD" - end, - Opts1 = - case Opts of - [Opt] -> Opt; - _ -> Opts - end, - io:format("~s(~w): ~w\n", [CtrlStr,Ctrl,Opts1]); - Any -> - io:format("Unexpected in cmd_dbg:~n~w~n",[Any]) - end; - true -> ok - end. + ok. +-endif. diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 4378ec5a52..f5069427a2 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -17,7 +17,7 @@ %% %CopyrightEnd% %% -%%% @doc Common Test Framework functions handlig test specifikations. +%%% @doc Common Test Framework functions handling test specifications. %%% %%% <p>This module exports functions that are used within CT to %%% scan and parse test specifikations.</p> @@ -185,7 +185,15 @@ prepare_cases(Node,Dir,Suite,Cases) -> {[{{Node,Dir},{Suite,all}}],SkipAll}; Skipped -> %% note: this adds a test even if only skip is specified - PrepC = lists:foldr(fun({C,{skip,_Cmt}},Acc) -> + PrepC = lists:foldr(fun({{G,Cs},{skip,_Cmt}}, Acc) when + is_atom(G) -> + case lists:keymember(G, 1, Cases) of + true -> + Acc; + false -> + [{skipped,G,Cs}|Acc] + end; + ({C,{skip,_Cmt}},Acc) -> case lists:member(C,Cases) of true -> Acc; @@ -194,7 +202,7 @@ prepare_cases(Node,Dir,Suite,Cases) -> end; (C,Acc) -> [C|Acc] end, [], Cases), - {{{Node,Dir},{Suite,PrepC}},Skipped} + {{{Node,Dir},{Suite,PrepC}},Skipped} end. get_skipped_suites(Node,Dir,Suites) -> @@ -210,7 +218,7 @@ get_skipped_cases(Node,Dir,Suite,Cases) -> case lists:keysearch(all,1,Cases) of {value,{all,{skip,Cmt}}} -> [{{Node,Dir},{Suite,Cmt}}]; - false -> + _ -> get_skipped_cases1(Node,Dir,Suite,Cases) end. @@ -270,33 +278,8 @@ collect_tests(Terms,TestSpec,Relaxed) -> put(relaxed,Relaxed), TestSpec1 = get_global(Terms,TestSpec), TestSpec2 = get_all_nodes(Terms,TestSpec1), - case catch evaluate(Terms,TestSpec2) of - {error,{Node,{M,F,A},Reason}} -> - io:format("Error! Common Test failed to evaluate ~w:~w/~w on ~w. " - "Reason: ~p~n~n", [M,F,A,Node,Reason]); - _ -> ok - end, - add_tests(Terms,TestSpec2). - -evaluate([{eval,NodeRef,{M,F,Args}}|Ts],Spec) -> - Node = ref2node(NodeRef,Spec#testspec.nodes), - case rpc:call(Node,M,F,Args) of - {badrpc,Reason} -> - throw({error,{Node,{M,F,length(Args)},Reason}}); - _ -> - ok - end, - evaluate(Ts,Spec); -evaluate([{eval,{M,F,Args}}|Ts],Spec) -> - case catch apply(M,F,Args) of - {'EXIT',Reason} -> - throw({error,{node(),{M,F,length(Args)},Reason}}); - _ -> - ok - end, - evaluate(Ts,Spec); -evaluate([],_Spec) -> - ok. + {Terms2, TestSpec3} = filter_init_terms(Terms, [], TestSpec2), + add_tests(Terms2,TestSpec3). get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) -> get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]}); @@ -305,6 +288,26 @@ get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) -> get_global([_|Ts],Spec) -> get_global(Ts,Spec); get_global([],Spec) -> Spec. +get_absfile(Callback, FullName,#testspec{spec_dir=SpecDir}) -> + % we need to temporary switch to new cwd here, because + % otherwise config files cannot be found + {ok, OldWd} = file:get_cwd(), + ok = file:set_cwd(SpecDir), + R = Callback:check_parameter(FullName), + ok = file:set_cwd(OldWd), + case R of + {ok, {file, FullName}}-> + File = filename:basename(FullName), + Dir = get_absname(filename:dirname(FullName),SpecDir), + filename:join(Dir,File); + {ok, {config, FullName}}-> + FullName; + {error, {nofile, FullName}}-> + FullName; + {error, {wrong_config, FullName}}-> + FullName + end. + get_absfile(FullName,#testspec{spec_dir=SpecDir}) -> File = filename:basename(FullName), Dir = get_absname(filename:dirname(FullName),SpecDir), @@ -353,6 +356,68 @@ get_all_nodes([_|Ts],Spec) -> get_all_nodes([],Spec) -> Spec. +filter_init_terms([{init, InitOptions}|Ts], NewTerms, Spec)-> + filter_init_terms([{init, list_nodes(Spec), InitOptions}|Ts], NewTerms, Spec); +filter_init_terms([{init, NodeRef, InitOptions}|Ts], NewTerms, Spec) + when is_atom(NodeRef)-> + filter_init_terms([{init, [NodeRef], InitOptions}|Ts], NewTerms, Spec); +filter_init_terms([{init, NodeRefs, InitOption}|Ts], NewTerms, Spec) when is_tuple(InitOption) -> + filter_init_terms([{init, NodeRefs, [InitOption]}|Ts], NewTerms, Spec); +filter_init_terms([{init, [NodeRef|NodeRefs], InitOptions}|Ts], NewTerms, Spec=#testspec{init=InitData})-> + NodeStartOptions = case lists:keyfind(node_start, 1, InitOptions) of + {node_start, NSOptions}-> + case lists:keyfind(callback_module, 1, NSOptions) of + {callback_module, _Callback}-> + NSOptions; + false-> + [{callback_module, ct_slave}|NSOptions] + end; + false-> + [] + end, + EvalTerms = case lists:keyfind(eval, 1, InitOptions) of + {eval, MFA} when is_tuple(MFA)-> + [MFA]; + {eval, MFAs} when is_list(MFAs)-> + MFAs; + false-> + [] + end, + Node = ref2node(NodeRef,Spec#testspec.nodes), + InitData2 = add_option({node_start, NodeStartOptions}, Node, InitData, true), + InitData3 = add_option({eval, EvalTerms}, Node, InitData2, false), + filter_init_terms([{init, NodeRefs, InitOptions}|Ts], NewTerms, Spec#testspec{init=InitData3}); +filter_init_terms([{init, [], _}|Ts], NewTerms, Spec)-> + filter_init_terms(Ts, NewTerms, Spec); +filter_init_terms([Term|Ts], NewTerms, Spec)-> + filter_init_terms(Ts, [Term|NewTerms], Spec); +filter_init_terms([], NewTerms, Spec)-> + {lists:reverse(NewTerms), Spec}. + +add_option([], _, List, _)-> + List; +add_option({Key, Value}, Node, List, WarnIfExists) when is_list(Value)-> + OldOptions = case lists:keyfind(Node, 1, List) of + {Node, Options}-> + Options; + false-> + [] + end, + NewOption = case lists:keyfind(Key, 1, OldOptions) of + {Key, OldOption} when WarnIfExists, OldOption/=[]-> + io:format("There is an option ~w=~w already defined for node ~p, skipping new ~w~n", + [Key, OldOption, Node, Value]), + OldOption; + {Key, OldOption}-> + OldOption ++ Value; + false-> + Value + end, + lists:keystore(Node, 1, List, + {Node, lists:keystore(Key, 1, OldOptions, {Key, NewOption})}); +add_option({Key, Value}, Node, List, WarnIfExists)-> + add_option({Key, [Value]}, Node, List, WarnIfExists). + save_nodes(Nodes,Spec=#testspec{nodes=NodeRefs}) -> NodeRefs1 = lists:foldr(fun(all_nodes,NR) -> @@ -375,6 +440,15 @@ save_nodes(Nodes,Spec=#testspec{nodes=NodeRefs}) -> list_nodes(#testspec{nodes=NodeRefs}) -> lists:map(fun({_Ref,Node}) -> Node end, NodeRefs). + + +%% --------------------------------------------------------- +%% / \ +%% | When adding tests, remember to update valid_terms/0 also! | +%% \ / +%% --------------------------------------------------------- + + %% Associate a "global" logdir with all nodes %% except those with specific logdir, e.g: %% ["/tmp/logdir",{ct1@finwe,"/tmp/logdir2"}] @@ -400,6 +474,24 @@ add_tests([{logdir,Node,Dir}|Ts],Spec) -> add_tests([{logdir,Dir}|Ts],Spec) -> add_tests([{logdir,all_nodes,Dir}|Ts],Spec); +%% --- label --- +add_tests([{label,all_nodes,Lbl}|Ts],Spec) -> + Labels = Spec#testspec.label, + Tests = [{label,N,Lbl} || N <- list_nodes(Spec), + lists:keymember(ref2node(N,Spec#testspec.nodes), + 1,Labels) == false], + add_tests(Tests++Ts,Spec); +add_tests([{label,Nodes,Lbl}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,label,[Lbl],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{label,Node,Lbl}|Ts],Spec) -> + Labels = Spec#testspec.label, + Labels1 = [{ref2node(Node,Spec#testspec.nodes),Lbl} | + lists:keydelete(ref2node(Node,Spec#testspec.nodes),1,Labels)], + add_tests(Ts,Spec#testspec{label=Labels1}); +add_tests([{label,Lbl}|Ts],Spec) -> + add_tests([{label,all_nodes,Lbl}|Ts],Spec); + %% --- cover --- add_tests([{cover,all_nodes,File}|Ts],Spec) -> Tests = lists:map(fun(N) -> {cover,N,File} end, list_nodes(Spec)), @@ -415,6 +507,36 @@ add_tests([{cover,Node,File}|Ts],Spec) -> add_tests([{cover,File}|Ts],Spec) -> add_tests([{cover,all_nodes,File}|Ts],Spec); +%% --- multiply_timetraps --- +add_tests([{multiply_timetraps,all_nodes,MT}|Ts],Spec) -> + Tests = lists:map(fun(N) -> {multiply_timetraps,N,MT} end, list_nodes(Spec)), + add_tests(Tests++Ts,Spec); +add_tests([{multiply_timetraps,Nodes,MT}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,multiply_timetraps,[MT],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{multiply_timetraps,Node,MT}|Ts],Spec) -> + MTs = Spec#testspec.multiply_timetraps, + MTs1 = [{ref2node(Node,Spec#testspec.nodes),MT} | + lists:keydelete(ref2node(Node,Spec#testspec.nodes),1,MTs)], + add_tests(Ts,Spec#testspec{multiply_timetraps=MTs1}); +add_tests([{multiply_timetraps,MT}|Ts],Spec) -> + add_tests([{multiply_timetraps,all_nodes,MT}|Ts],Spec); + +%% --- scale_timetraps --- +add_tests([{scale_timetraps,all_nodes,ST}|Ts],Spec) -> + Tests = lists:map(fun(N) -> {scale_timetraps,N,ST} end, list_nodes(Spec)), + add_tests(Tests++Ts,Spec); +add_tests([{scale_timetraps,Nodes,ST}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,scale_timetraps,[ST],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{scale_timetraps,Node,ST}|Ts],Spec) -> + STs = Spec#testspec.scale_timetraps, + STs1 = [{ref2node(Node,Spec#testspec.nodes),ST} | + lists:keydelete(ref2node(Node,Spec#testspec.nodes),1,STs)], + add_tests(Ts,Spec#testspec{scale_timetraps=STs1}); +add_tests([{scale_timetraps,ST}|Ts],Spec) -> + add_tests([{scale_timetraps,all_nodes,ST}|Ts],Spec); + %% --- config --- add_tests([{config,all_nodes,Files}|Ts],Spec) -> Tests = lists:map(fun(N) -> {config,N,Files} end, list_nodes(Spec)), @@ -434,6 +556,27 @@ add_tests([{config,Node,F}|Ts],Spec) -> add_tests([{config,Files}|Ts],Spec) -> add_tests([{config,all_nodes,Files}|Ts],Spec); + +%% --- userconfig --- +add_tests([{userconfig,all_nodes,CBF}|Ts],Spec) -> + Tests = lists:map(fun(N) -> {userconfig,N,CBF} end, list_nodes(Spec)), + add_tests(Tests++Ts,Spec); +add_tests([{userconfig,Nodes,CBF}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,userconfig,[CBF],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{userconfig,Node,[{Callback, Config}|CBF]}|Ts],Spec) -> + Cfgs = Spec#testspec.userconfig, + Node1 = ref2node(Node,Spec#testspec.nodes), + add_tests([{userconfig,Node,CBF}|Ts], + Spec#testspec{userconfig=[{Node1,{Callback, + get_absfile(Callback, Config ,Spec)}}|Cfgs]}); +add_tests([{userconfig,_Node,[]}|Ts],Spec) -> + add_tests(Ts,Spec); +add_tests([{userconfig,Node,CBF}|Ts],Spec) -> + add_tests([{userconfig,Node,[CBF]}|Ts],Spec); +add_tests([{userconfig,CBF}|Ts],Spec) -> + add_tests([{userconfig,all_nodes,CBF}|Ts],Spec); + %% --- event_handler --- add_tests([{event_handler,all_nodes,Hs}|Ts],Spec) -> Tests = lists:map(fun(N) -> {event_handler,N,Hs,[]} end, list_nodes(Spec)), @@ -516,6 +659,38 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) -> Ss,Tests), add_tests(Ts,Spec#testspec{tests=Tests1}); +%% --- groups --- +%% Later make it possible to specify group execution properties +%% that will override thse in the suite. Also make it possible +%% create dynamic groups in specification, i.e. to group test cases +%% by means of groups defined only in the test specification. +add_tests([{groups,all_nodes,Dir,Suite,Gs}|Ts],Spec) -> + add_tests([{groups,list_nodes(Spec),Dir,Suite,Gs}|Ts],Spec); +add_tests([{groups,all_nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> + add_tests([{groups,list_nodes(Spec),Dir,Suite,Gs,{cases,TCs}}|Ts],Spec); +add_tests([{groups,Dir,Suite,Gs}|Ts],Spec) -> + add_tests([{groups,all_nodes,Dir,Suite,Gs}|Ts],Spec); +add_tests([{groups,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> + add_tests([{groups,all_nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec); +add_tests([{groups,Nodes,Dir,Suite,Gs}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,groups,[Dir,Suite,Gs],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{groups,Nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,groups,[Dir,Suite,Gs,{cases,TCs}],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) -> + Tests = Spec#testspec.tests, + Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), + ref2dir(Dir,Spec#testspec.alias), + Suite,Gs,all,Tests), + add_tests(Ts,Spec#testspec{tests=Tests1}); +add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> + Tests = Spec#testspec.tests, + Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes), + ref2dir(Dir,Spec#testspec.alias), + Suite,Gs,TCs,Tests), + add_tests(Ts,Spec#testspec{tests=Tests1}); + %% --- cases --- add_tests([{cases,all_nodes,Dir,Suite,Cs}|Ts],Spec) -> add_tests([{cases,list_nodes(Spec),Dir,Suite,Cs}|Ts],Spec); @@ -546,6 +721,34 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) -> Ss,Cmt,Tests), add_tests(Ts,Spec#testspec{tests=Tests1}); +%% --- skip_groups --- +add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,Cmt}|Ts],Spec) -> + add_tests([{skip_groups,list_nodes(Spec),Dir,Suite,Gs,Cmt}|Ts],Spec); +add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> + add_tests([{skip_groups,list_nodes(Spec),Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec); +add_tests([{skip_groups,Dir,Suite,Gs,Cmt}|Ts],Spec) -> + add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,Cmt}|Ts],Spec); +add_tests([{skip_groups,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> + add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec); +add_tests([{skip_groups,Nodes,Dir,Suite,Gs,Cmt}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,Cmt],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{skip_groups,Nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,{cases,TCs},Cmt],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) -> + Tests = Spec#testspec.tests, + Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), + ref2dir(Dir,Spec#testspec.alias), + Suite,Gs,all,Cmt,Tests), + add_tests(Ts,Spec#testspec{tests=Tests1}); +add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> + Tests = Spec#testspec.tests, + Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes), + ref2dir(Dir,Spec#testspec.alias), + Suite,Gs,TCs,Cmt,Tests), + add_tests(Ts,Spec#testspec{tests=Tests1}); + %% --- skip_cases --- add_tests([{skip_cases,all_nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) -> add_tests([{skip_cases,list_nodes(Spec),Dir,Suite,Cs,Cmt}|Ts],Spec); @@ -614,8 +817,11 @@ separate([],_,_,_) -> %% Representation: -%% {{Node,Dir},[{Suite1,[case11,case12,...]},{Suite2,[case21,case22,...]},...]} -%% {{Node,Dir},[{Suite1,{skip,Cmt}},{Suite2,[{case21,{skip,Cmt}},case22,...]},...]} +%% {{Node,Dir},[{Suite1,[GrOrCase11,GrOrCase12,...]}, +%% {Suite2,[GrOrCase21,GrOrCase22,...]},...]} +%% {{Node,Dir},[{Suite1,{skip,Cmt}}, +%% {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]} +%% GrOrCase = {GroupName,[Case1,Case2,...]} | Case insert_suites(Node,Dir,[S|Ss],Tests) -> Tests1 = insert_cases(Node,Dir,S,all,Tests), @@ -625,6 +831,54 @@ insert_suites(_Node,_Dir,[],Tests) -> insert_suites(Node,Dir,S,Tests) -> insert_suites(Node,Dir,[S],Tests). +insert_groups(Node,Dir,Suite,Group,Cases,Tests) when is_atom(Group) -> + insert_groups(Node,Dir,Suite,[Group],Cases,Tests); +insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + case lists:keysearch({Node,Dir},1,Tests) of + {value,{{Node,Dir},[{all,_}]}} -> + Tests; + {value,{{Node,Dir},Suites0}} -> + Suites1 = insert_groups1(Suite, + [{Gr,Cases} || Gr <- Groups], + Suites0), + insert_in_order({{Node,Dir},Suites1},Tests); + false -> + Groups1 = [{Gr,Cases} || Gr <- Groups], + insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests) + end; +insert_groups(Node,Dir,Suite,Groups,Case,Tests) when is_atom(Case) -> + Cases = if Case == all -> all; true -> [Case] end, + insert_groups(Node,Dir,Suite,Groups,Cases,Tests). + +insert_groups1(_Suite,_Groups,all) -> + all; +insert_groups1(Suite,Groups,Suites0) -> + case lists:keysearch(Suite,1,Suites0) of + {value,{Suite,all}} -> + Suites0; + {value,{Suite,GrAndCases0}} -> + GrAndCases = insert_groups2(Groups,GrAndCases0), + insert_in_order({Suite,GrAndCases},Suites0); + false -> + insert_in_order({Suite,Groups},Suites0) + end. + +insert_groups2(_Groups,all) -> + all; +insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) -> + case lists:keysearch(GrName,1,GrAndCases) of + {value,{GrName,all}} -> + GrAndCases; + {value,{GrName,Cases0}} -> + Cases1 = insert_in_order(Cases,Cases0), + insert_groups2(Groups,insert_in_order({GrName,Cases1},GrAndCases)); + false -> + insert_groups2(Groups,insert_in_order(Group,GrAndCases)) + end; +insert_groups2([],GrAndCases) -> + GrAndCases. + insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) -> case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> @@ -659,6 +913,40 @@ skip_suites(_Node,_Dir,[],_Cmt,Tests) -> skip_suites(Node,Dir,S,Cmt,Tests) -> skip_suites(Node,Dir,[S],Cmt,Tests). +skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); +skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case), + Case =/= all -> + skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); +skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when + ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Suites = + case lists:keysearch({Node,Dir},1,Tests) of + {value,{{Node,Dir},Suites0}} -> + Suites0; + false -> + [] + end, + Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites), + insert_in_order({{Node,Dir},Suites1},Tests); +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> + Cases = if Case == all -> all; true -> [Case] end, + skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests). + +skip_groups1(Suite,Groups,Cmt,Suites0) -> + SkipGroups = lists:map(fun(Group) -> + {Group,{skip,Cmt}} + end,Groups), + case lists:keysearch(Suite,1,Suites0) of + {value,{Suite,GrAndCases0}} -> + GrAndCases1 = GrAndCases0 ++ SkipGroups, + insert_in_order({Suite,GrAndCases1},Suites0); + false -> + insert_in_order({Suite,SkipGroups},Suites0) + end. + skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) -> Suites = case lists:keysearch({Node,Dir},1,Tests) of @@ -753,21 +1041,34 @@ valid_terms() -> {cover,3}, {config,2}, {config,3}, + {userconfig,2}, + {userconfig,3}, {alias,3}, {logdir,2}, {logdir,3}, + {label,2}, + {label,3}, {event_handler,2}, {event_handler,3}, {event_handler,4}, + {multiply_timetraps,2}, + {multiply_timetraps,3}, + {scale_timetraps,2}, + {scale_timetraps,3}, {include,2}, {include,3}, - {suites,3}, {suites,4}, + {groups,4}, + {groups,5}, + {groups,6}, {cases,4}, {cases,5}, {skip_suites,4}, {skip_suites,5}, + {skip_groups,5}, + {skip_groups,6}, + {skip_groups,7}, {skip_cases,5}, {skip_cases,6} ]. @@ -816,7 +1117,3 @@ common_letters([L|Ls],Term,Count) -> end; common_letters([],_,Count) -> Count. - - - - diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index ba3d789f8d..0a434666fa 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -30,10 +30,7 @@ -export([register_connection/4,unregister_connection/1, does_connection_exist/3,get_key_from_name/1]). --export([require/1, require/2, get_config/1, get_config/2, get_config/3, - set_default_config/2, set_default_config/3, delete_default_config/1, - get_all_config/0, update_config/2, - release_allocated/0, close_connections/0]). +-export([close_connections/0]). -export([save_suite_data/3, save_suite_data/2, read_suite_data/1, delete_suite_data/0, delete_suite_data/1, match_delete_suite_data/1, @@ -46,6 +43,8 @@ silence_all_connections/0, silence_connections/1, is_silenced/1, reset_silent_connections/0]). +-export([get_mode/0, create_table/3, read_opts/0]). + -export([set_cwd/1, reset_cwd/0]). -export([parse_table/1]). @@ -56,23 +55,15 @@ -export([is_test_dir/1, get_testdir/2]). --export([encrypt_config_file/2, encrypt_config_file/3, - decrypt_config_file/2, decrypt_config_file/3]). - --export([kill_attached/2, get_attached/1]). +-export([kill_attached/2, get_attached/1, ct_make_ref/0]). -export([warn_duplicates/1]). -include("ct_event.hrl"). -include("ct_util.hrl"). --record(ct_conf,{key,value,ref,name='_UNDEF',default=false}). -%% default = {true,suite} | {true,testcase} | false - -record(suite_data, {key,name,value}). --define(cryptfile, ".ct_config.crypt"). - %%%----------------------------------------------------------------- %%% @spec start(Mode) -> Pid | exit(Error) %%% Mode = normal | interactive @@ -119,7 +110,6 @@ start(Mode,LogDir) -> do_start(Parent,Mode,LogDir) -> process_flag(trap_exit,true), register(ct_util_server,self()), - create_table(?attr_table,bag,#ct_conf.key), create_table(?conn_table,#conn.handle), create_table(?board_table,2), create_table(?suite_table,#suite_data.key), @@ -135,7 +125,6 @@ do_start(Parent,Mode,LogDir) -> Parent ! {self(),Error}, exit(Error) end, - %% start an event manager (if not already started by master) case ct_event:start_link() of {error,{already_started,_}} -> @@ -148,38 +137,34 @@ do_start(Parent,Mode,LogDir) -> ct_event:add_handler([{vts,VtsPid}]) end end, - case read_config_files(Opts) of - ok -> - %% add user handlers - case lists:keysearch(event_handler,1,Opts) of - {value,{_,Handlers}} -> - Add = fun({H,Args}) -> - case catch gen_event:add_handler(?CT_EVMGR_REF,H,Args) of - ok -> ok; - {'EXIT',Why} -> exit(Why); - Other -> exit({event_handler,Other}) - end - end, - case catch lists:foreach(Add,Handlers) of - {'EXIT',Reason} -> - Parent ! {self(),Reason}; - _ -> - ok - end; - false -> + %% start ct_config server + ct_config:start(Mode), + %% add user event handlers + case lists:keysearch(event_handler,1,Opts) of + {value,{_,Handlers}} -> + Add = fun({H,Args}) -> + case catch gen_event:add_handler(?CT_EVMGR_REF,H,Args) of + ok -> ok; + {'EXIT',Why} -> exit(Why); + Other -> exit({event_handler,Other}) + end + end, + case catch lists:foreach(Add,Handlers) of + {'EXIT',Reason} -> + Parent ! {self(),Reason}; + _ -> ok - end, - {StartTime,TestLogDir} = ct_logs:init(Mode), - ct_event:notify(#event{name=test_start, - node=node(), - data={StartTime, - lists:flatten(TestLogDir)}}), - Parent ! {self(),started}, - loop(Mode,[],StartDir); - ReadError -> - Parent ! {self(),ReadError}, - exit(ReadError) - end. + end; + false -> + ok + end, + {StartTime,TestLogDir} = ct_logs:init(Mode), + ct_event:notify(#event{name=test_start, + node=node(), + data={StartTime, + lists:flatten(TestLogDir)}}), + Parent ! {self(),started}, + loop(Mode,[],StartDir). create_table(TableName,KeyPos) -> create_table(TableName,set,KeyPos). @@ -197,106 +182,6 @@ read_opts() -> {error,{bad_installation,Error}} end. -read_config_files(Opts) -> - ConfigFiles = - lists:foldl(fun({config,Files},Acc) -> - Acc ++ Files; - (_,Acc) -> - Acc - end,[],Opts), - read_config_files1(ConfigFiles). - -read_config_files1([ConfigFile|Files]) -> - case file:consult(ConfigFile) of - {ok,Config} -> - set_config(Config), - read_config_files1(Files); - {error,enoent} -> - {user_error,{config_file_error,ConfigFile,enoent}}; - {error,Reason} -> - Key = - case application:get_env(common_test, decrypt) of - {ok,KeyOrFile} -> - case KeyOrFile of - {key,K} -> - K; - {file,F} -> - get_crypt_key_from_file(F) - end; - _ -> - get_crypt_key_from_file() - end, - case Key of - {error,no_crypt_file} -> - {user_error,{config_file_error,ConfigFile,Reason}}; - {error,CryptError} -> - {user_error,{decrypt_file_error,ConfigFile,CryptError}}; - _ when is_list(Key) -> - case decrypt_config_file(ConfigFile, undefined, {key,Key}) of - {ok,CfgBin} -> - case read_config_terms(CfgBin) of - {error,ReadFail} -> - {user_error,{config_file_error,ConfigFile,ReadFail}}; - Config -> - set_config(Config), - read_config_files1(Files) - end; - {error,DecryptFail} -> - {user_error,{decrypt_config_error,ConfigFile,DecryptFail}} - end; - _ -> - {user_error,{bad_decrypt_key,ConfigFile,Key}} - end - end; -read_config_files1([]) -> - ok. - -read_config_terms(Bin) when is_binary(Bin) -> - case catch binary_to_list(Bin) of - {'EXIT',_} -> - {error,invalid_textfile}; - Lines -> - read_config_terms(Lines) - end; -read_config_terms(Lines) when is_list(Lines) -> - read_config_terms1(erl_scan:tokens([], Lines, 0), 1, [], []). - -read_config_terms1({done,{ok,Ts,EL},Rest}, L, Terms, _) -> - case erl_parse:parse_term(Ts) of - {ok,Term} when Rest == [] -> - lists:reverse([Term|Terms]); - {ok,Term} -> - read_config_terms1(erl_scan:tokens([], Rest, 0), - EL+1, [Term|Terms], Rest); - _ -> - {error,{bad_term,{L,EL}}} - end; -read_config_terms1({done,{eof,_},_}, _, Terms, Rest) when Rest == [] -> - lists:reverse(Terms); -read_config_terms1({done,{eof,EL},_}, L, _, _) -> - {error,{bad_term,{L,EL}}}; -read_config_terms1({done,{error,Info,EL},_}, L, _, _) -> - {error,{Info,{L,EL}}}; -read_config_terms1({more,_}, L, Terms, Rest) -> - case string:tokens(Rest, [$\n,$\r,$\t]) of - [] -> - lists:reverse(Terms); - _ -> - {error,{bad_term,L}} - end. - -set_default_config(NewConfig, Scope) -> - call({set_default_config, {NewConfig, Scope}}). - -set_default_config(Name, NewConfig, Scope) -> - call({set_default_config, {Name, NewConfig, Scope}}). - -delete_default_config(Scope) -> - call({delete_default_config, Scope}). - -update_config(Name, Config) -> - call({update_config, {Name, Config}}). - save_suite_data(Key, Value) -> call({save_suite_data, {Key, undefined, Value}}). @@ -342,26 +227,6 @@ loop(Mode,TestData,StartDir) -> ct_logs:make_last_run_index(), return(From,ok), loop(Mode,TestData,StartDir); - {{require,Name,Tag,SubTags},From} -> - Result = do_require(Name,Tag,SubTags), - return(From,Result), - loop(Mode,TestData,StartDir); - {{set_default_config,{Config,Scope}},From} -> - set_config(Config,{true,Scope}), - return(From,ok), - loop(Mode,TestData,StartDir); - {{set_default_config,{Name,Config,Scope}},From} -> - set_config(Name,Config,{true,Scope}), - return(From,ok), - loop(Mode,TestData,StartDir); - {{delete_default_config,Scope},From} -> - delete_config({true,Scope}), - return(From,ok), - loop(Mode,TestData,StartDir); - {{update_config,{Name,NewConfig}},From} -> - update_conf(Name,NewConfig), - return(From,ok), - loop(Mode,TestData,StartDir); {{save_suite_data,{Key,Name,Value}},From} -> ets:insert(?suite_table, #suite_data{key=Key, name=Name, @@ -434,14 +299,14 @@ loop(Mode,TestData,StartDir) -> ct_event:sync_notify(#event{name=test_done, node=node(), data=Time}), - ets:delete(?attr_table), close_connections(ets:tab2list(?conn_table)), ets:delete(?conn_table), ets:delete(?board_table), ets:delete(?suite_table), ct_logs:close(How), - file:set_cwd(StartDir), ct_event:stop(), + ct_config:stop(), + file:set_cwd(StartDir), return(From,ok); {get_mode,From} -> return(From,Mode), @@ -463,6 +328,8 @@ close_connections([#conn{handle=Handle,callback=CB}|Conns]) -> close_connections([]) -> ok. +get_key_from_name(Name)-> + ct_config:get_key_from_name(Name). %%%----------------------------------------------------------------- %%% @spec register_connection(TargetName,Address,Callback,Handle) -> @@ -480,7 +347,7 @@ close_connections([]) -> %%% test is finished by calling <code>Callback:close/1</code>.</p> register_connection(TargetName,Address,Callback,Handle) -> TargetRef = - case get_ref_from_name(TargetName) of + case ct_config:get_ref_from_name(TargetName) of {ok,Ref} -> Ref; _ -> @@ -518,7 +385,7 @@ unregister_connection(Handle) -> %%% %%% @doc Check if a connection already exists. does_connection_exist(TargetName,Address,Callback) -> - case get_ref_from_name(TargetName) of + case ct_config:get_ref_from_name(TargetName) of {ok,TargetRef} -> case ets:select(?conn_table,[{#conn{handle='$1', targetref=TargetRef, @@ -548,7 +415,7 @@ does_connection_exist(TargetName,Address,Callback) -> %%% @doc Return all connections for the <code>Callback</code> on the %%% given target (<code>TargetName</code>). get_connections(TargetName,Callback) -> - case get_ref_from_name(TargetName) of + case ct_config:get_ref_from_name(TargetName) of {ok,Ref} -> {ok,ets:select(?conn_table,[{#conn{handle='$1', address='$2', @@ -568,250 +435,11 @@ get_target_name(ConnPid) -> [], ['$1']}]) of [TargetRef] -> - get_name_from_ref(TargetRef); + ct_config:get_name_from_ref(TargetRef); [] -> {error,{unknown_connection,ConnPid}} end. - -%%%----------------------------------------------------------------- -%%% @hidden -%%% @equiv ct:require/1 -require(Key) when is_atom(Key) -> - require({Key,[]}); -require({Key,SubKeys}) when is_atom(Key) -> - allocate('_UNDEF',Key,to_list(SubKeys)); -require(Key) -> - {error,{invalid,Key}}. - - -%%%----------------------------------------------------------------- -%%% @hidden -%%% @equiv ct:require/2 -require(Name,Key) when is_atom(Key) -> - require(Name,{Key,[]}); -require(Name,{Key,SubKeys}) when is_atom(Name), is_atom(Key) -> - call({require,Name,Key,to_list(SubKeys)}); -require(Name,Keys) -> - {error,{invalid,{Name,Keys}}}. - -to_list(X) when is_list(X) -> X; -to_list(X) -> [X]. - -do_require(Name,Key,SubKeys) when is_list(SubKeys) -> - case get_key_from_name(Name) of - {error,_} -> - allocate(Name,Key,SubKeys); - {ok,Key} -> - %% already allocated - check that it has all required subkeys - Vals = [Val || {_Ref,Val} <- lookup_name(Name)], - case get_subconfig(SubKeys,Vals) of - {ok,_SubMapped} -> - ok; - Error -> - Error - end; - {ok,OtherKey} -> - {error,{name_in_use,Name,OtherKey}} - end. - -allocate(Name,Key,SubKeys) -> - case ets:match_object(?attr_table,#ct_conf{key=Key,name='_UNDEF',_='_'}) of - [] -> - {error,{not_available,Key}}; - Available -> - case allocate_subconfig(Name,SubKeys,Available,false) of - ok -> - ok; - Error -> - Error - end - end. - -allocate_subconfig(Name,SubKeys,[C=#ct_conf{value=Value}|Rest],Found) -> - case do_get_config(SubKeys,Value,[]) of - {ok,_SubMapped} -> - ets:insert(?attr_table,C#ct_conf{name=Name}), - allocate_subconfig(Name,SubKeys,Rest,true); - _Error -> - allocate_subconfig(Name,SubKeys,Rest,Found) - end; -allocate_subconfig(_Name,_SubKeys,[],true) -> - ok; -allocate_subconfig(_Name,SubKeys,[],false) -> - {error,{not_available,SubKeys}}. - - - - -%%%----------------------------------------------------------------- -%%% @hidden -%%% @equiv ct:get_config/1 -get_config(KeyOrName) -> - get_config(KeyOrName,undefined,[]). - -%%%----------------------------------------------------------------- -%%% @hidden -%%% @equiv ct:get_config/2 -get_config(KeyOrName,Default) -> - get_config(KeyOrName,Default,[]). - -%%%----------------------------------------------------------------- -%%% @hidden -%%% @equiv ct:get_config/3 -get_config(KeyOrName,Default,Opts) when is_atom(KeyOrName) -> - case lookup_config(KeyOrName) of - [] -> - Default; - [{_Ref,Val}|_] = Vals -> - case {lists:member(all,Opts),lists:member(element,Opts)} of - {true,true} -> - [{KeyOrName,V} || {_R,V} <- lists:sort(Vals)]; - {true,false} -> - [V || {_R,V} <- lists:sort(Vals)]; - {false,true} -> - {KeyOrName,Val}; - {false,false} -> - Val - end - end; - -get_config({KeyOrName,SubKey},Default,Opts) -> - case lookup_config(KeyOrName) of - [] -> - Default; - Vals -> - Vals1 = case [Val || {_Ref,Val} <- lists:sort(Vals)] of - Result=[L|_] when is_list(L) -> - case L of - [{_,_}|_] -> - Result; - _ -> - [] - end; - _ -> - [] - end, - case get_subconfig([SubKey],Vals1,[],Opts) of - {ok,[{_,SubVal}|_]=SubVals} -> - case {lists:member(all,Opts),lists:member(element,Opts)} of - {true,true} -> - [{{KeyOrName,SubKey},Val} || {_,Val} <- SubVals]; - {true,false} -> - [Val || {_SubKey,Val} <- SubVals]; - {false,true} -> - {{KeyOrName,SubKey},SubVal}; - {false,false} -> - SubVal - end; - _ -> - Default - end - end. - - -get_subconfig(SubKeys,Values) -> - get_subconfig(SubKeys,Values,[],[]). - -get_subconfig(SubKeys,[Value|Rest],Mapped,Opts) -> - case do_get_config(SubKeys,Value,[]) of - {ok,SubMapped} -> - case lists:member(all,Opts) of - true -> - get_subconfig(SubKeys,Rest,Mapped++SubMapped,Opts); - false -> - {ok,SubMapped} - end; - _Error -> - get_subconfig(SubKeys,Rest,Mapped,Opts) - end; -get_subconfig(SubKeys,[],[],_) -> - {error,{not_available,SubKeys}}; -get_subconfig(_SubKeys,[],Mapped,_) -> - {ok,Mapped}. - -do_get_config([Key|Required],Available,Mapped) -> - case lists:keysearch(Key,1,Available) of - {value,{Key,Value}} -> - NewAvailable = lists:keydelete(Key,1,Available), - NewMapped = [{Key,Value}|Mapped], - do_get_config(Required,NewAvailable,NewMapped); - false -> - {error,{not_available,Key}} - end; -do_get_config([],_Available,Mapped) -> - {ok,lists:reverse(Mapped)}. - -get_all_config() -> - ets:select(?attr_table,[{#ct_conf{name='$1',key='$2',value='$3', - default='$4',_='_'}, - [], - [{{'$1','$2','$3','$4'}}]}]). - -lookup_config(KeyOrName) -> - case lookup_name(KeyOrName) of - [] -> - lookup_key(KeyOrName); - Values -> - Values - end. - -lookup_name(Name) -> - ets:select(?attr_table,[{#ct_conf{ref='$1',value='$2',name=Name,_='_'}, - [], - [{{'$1','$2'}}]}]). -lookup_key(Key) -> - ets:select(?attr_table,[{#ct_conf{key=Key,ref='$1',value='$2',name='_UNDEF',_='_'}, - [], - [{{'$1','$2'}}]}]). - -set_config(Config) -> - set_config('_UNDEF',Config,false). - -set_config(Config,Default) -> - set_config('_UNDEF',Config,Default). - -set_config(Name,Config,Default) -> - [ets:insert(?attr_table, - #ct_conf{key=Key,value=Val,ref=ct_make_ref(), - name=Name,default=Default}) || - {Key,Val} <- Config]. - -delete_config(Default) -> - ets:match_delete(?attr_table,#ct_conf{default=Default,_='_'}), - ok. - - -%%%----------------------------------------------------------------- -%%% @spec release_allocated() -> ok -%%% -%%% @doc Release all allocated resources, but don't take down any -%%% connections. -release_allocated() -> - Allocated = ets:select(?attr_table,[{#ct_conf{name='$1',_='_'}, - [{'=/=','$1','_UNDEF'}], - ['$_']}]), - release_allocated(Allocated). -release_allocated([H|T]) -> - ets:delete_object(?attr_table,H), - ets:insert(?attr_table,H#ct_conf{name='_UNDEF'}), - release_allocated(T); -release_allocated([]) -> - ok. - -%%%----------------------------------------------------------------- -%%% @spec -%%% -%%% @doc -update_conf(Name, NewConfig) -> - Old = ets:select(?attr_table,[{#ct_conf{name=Name,_='_'},[],['$_']}]), - lists:foreach(fun(OldElem) -> - NewElem = OldElem#ct_conf{value=NewConfig}, - ets:delete_object(?attr_table, OldElem), - ets:insert(?attr_table, NewElem) - end, Old), - ok. - %%%----------------------------------------------------------------- %%% @spec close_connections() -> ok %%% @@ -995,167 +623,6 @@ get_testdir(Dir, _) -> %%% @spec %%% %%% @doc -encrypt_config_file(SrcFileName, EncryptFileName) -> - case get_crypt_key_from_file() of - {error,_} = E -> - E; - Key -> - encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) - end. - -%%%----------------------------------------------------------------- -%%% @spec -%%% -%%% @doc -encrypt_config_file(SrcFileName, EncryptFileName, {file,KeyFile}) -> - case get_crypt_key_from_file(KeyFile) of - {error,_} = E -> - E; - Key -> - encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) - end; - -encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) -> - crypto:start(), - {K1,K2,K3,IVec} = make_crypto_key(Key), - case file:read_file(SrcFileName) of - {ok,Bin0} -> - Bin1 = term_to_binary({SrcFileName,Bin0}), - Bin2 = case byte_size(Bin1) rem 8 of - 0 -> Bin1; - N -> list_to_binary([Bin1,random_bytes(8-N)]) - end, - EncBin = crypto:des3_cbc_encrypt(K1, K2, K3, IVec, Bin2), - case file:write_file(EncryptFileName, EncBin) of - ok -> - io:format("~s --(encrypt)--> ~s~n", - [SrcFileName,EncryptFileName]), - ok; - {error,Reason} -> - {error,{Reason,EncryptFileName}} - end; - {error,Reason} -> - {error,{Reason,SrcFileName}} - end. - -%%%----------------------------------------------------------------- -%%% @spec -%%% -%%% @doc -decrypt_config_file(EncryptFileName, TargetFileName) -> - case get_crypt_key_from_file() of - {error,_} = E -> - E; - Key -> - decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) - end. - - -%%%----------------------------------------------------------------- -%%% @spec -%%% -%%% @doc -decrypt_config_file(EncryptFileName, TargetFileName, {file,KeyFile}) -> - case get_crypt_key_from_file(KeyFile) of - {error,_} = E -> - E; - Key -> - decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) - end; - -decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) -> - crypto:start(), - {K1,K2,K3,IVec} = make_crypto_key(Key), - case file:read_file(EncryptFileName) of - {ok,Bin} -> - DecBin = crypto:des3_cbc_decrypt(K1, K2, K3, IVec, Bin), - case catch binary_to_term(DecBin) of - {'EXIT',_} -> - {error,bad_file}; - {_SrcFile,SrcBin} -> - case TargetFileName of - undefined -> - {ok,SrcBin}; - _ -> - case file:write_file(TargetFileName, SrcBin) of - ok -> - io:format("~s --(decrypt)--> ~s~n", - [EncryptFileName,TargetFileName]), - ok; - {error,Reason} -> - {error,{Reason,TargetFileName}} - end - end - end; - {error,Reason} -> - {error,{Reason,EncryptFileName}} - end. - - -get_crypt_key_from_file(File) -> - case file:read_file(File) of - {ok,Bin} -> - case catch string:tokens(binary_to_list(Bin), [$\n,$\r]) of - [Key] -> - Key; - _ -> - {error,{bad_crypt_file,File}} - end; - {error,Reason} -> - {error,{Reason,File}} - end. - -get_crypt_key_from_file() -> - CwdFile = filename:join(".",?cryptfile), - {Result,FullName} = - case file:read_file(CwdFile) of - {ok,Bin} -> - {Bin,CwdFile}; - _ -> - case init:get_argument(home) of - {ok,[[Home]]} -> - HomeFile = filename:join(Home,?cryptfile), - case file:read_file(HomeFile) of - {ok,Bin} -> - {Bin,HomeFile}; - _ -> - {{error,no_crypt_file},noent} - end; - _ -> - {{error,no_crypt_file},noent} - end - end, - case FullName of - noent -> - Result; - _ -> - case catch string:tokens(binary_to_list(Result), [$\n,$\r]) of - [Key] -> - io:format("~nCrypt key file: ~s~n", [FullName]), - Key; - _ -> - {error,{bad_crypt_file,FullName}} - end - end. - -make_crypto_key(String) -> - <<K1:8/binary,K2:8/binary>> = First = erlang:md5(String), - <<K3:8/binary,IVec:8/binary>> = erlang:md5([First|lists:reverse(String)]), - {K1,K2,K3,IVec}. - -random_bytes(N) -> - {A,B,C} = now(), - random:seed(A, B, C), - random_bytes_1(N, []). - -random_bytes_1(0, Acc) -> Acc; -random_bytes_1(N, Acc) -> random_bytes_1(N-1, [random:uniform(255)|Acc]). - - -%%%----------------------------------------------------------------- -%%% @spec -%%% -%%% @doc get_attached(TCPid) -> case dbg_iserver:safe_call({get_attpid,TCPid}) of {ok,AttPid} when is_pid(AttPid) -> @@ -1210,7 +677,7 @@ call(Msg) -> ct_util_server ! {Msg,{self(),Ref}}, receive {Ref, Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), Result; {'DOWN',MRef,process,_,Reason} -> {error,{ct_util_server_down,Reason}} @@ -1244,37 +711,6 @@ ct_make_ref_loop(N) -> From ! {self(),N}, ct_make_ref_loop(N+1) end. - -get_ref_from_name(Name) -> - case ets:select(?attr_table,[{#ct_conf{name=Name,ref='$1',_='_'}, - [], - ['$1']}]) of - [Ref] -> - {ok,Ref}; - _ -> - {error,{no_such_name,Name}} - end. - -get_name_from_ref(Ref) -> - case ets:select(?attr_table,[{#ct_conf{name='$1',ref=Ref,_='_'}, - [], - ['$1']}]) of - [Name] -> - {ok,Name}; - _ -> - {error,{no_such_ref,Ref}} - end. - -get_key_from_name(Name) -> - case ets:select(?attr_table,[{#ct_conf{name=Name,key='$1',_='_'}, - [], - ['$1']}]) of - [Key|_] -> - {ok,Key}; - _ -> - {error,{no_such_name,Name}} - end. - abs_name(Dir0) -> Abs = filename:absname(Dir0), diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index c1dc14f943..ee973f6220 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -29,11 +29,16 @@ -record(testspec, {spec_dir, nodes=[], + init=[], + label=[], logdir=["."], cover=[], config=[], + userconfig=[], event_handler=[], include=[], + multiply_timetraps=[], + scale_timetraps=[], alias=[], tests=[]}). @@ -50,3 +55,4 @@ -define(CT_MEVMGR_REF, ct_master_event). -define(missing_suites_info, "missing_suites.info"). +-define(ct_config_txt, ct_config_plain). diff --git a/lib/common_test/src/vts.erl b/lib/common_test/src/vts.erl index ad4845a7c3..2ee982d726 100644 --- a/lib/common_test/src/vts.erl +++ b/lib/common_test/src/vts.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -100,7 +100,7 @@ start_link() -> MRef = erlang:monitor(process,Pid), receive {Pid,started} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), {ok,Pid}; {'DOWN',MRef,process,_,Reason} -> {error,{vts,died,Reason}} @@ -160,11 +160,13 @@ init(Parent) -> loop(State) -> receive - {{init_data,ConfigFiles,EvHandlers,LogDir,Tests},From} -> - ct_install(State), + {{init_data,Config,EvHandlers,LogDir,Tests},From} -> + %% ct:pal("State#state.current_log_dir=~p", [State#state.current_log_dir]), + NewState = State#state{config=Config,event_handler=EvHandlers, + current_log_dir=LogDir,tests=Tests}, + ct_install(NewState), return(From,ok), - loop(#state{config=ConfigFiles,event_handler=EvHandlers, - current_log_dir=LogDir,tests=Tests}); + loop(NewState); {start_page,From} -> return(From,start_page1()), loop(State); @@ -257,7 +259,7 @@ call(Msg) -> Pid ! {Msg,{self(),Ref}}, receive {Ref, Result} -> - erlang:demonitor(MRef), + erlang:demonitor(MRef, [flush]), Result; {'DOWN',MRef,process,_,Reason} -> {error,{process_down,Pid,Reason}} diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 35ba22aa59..e3dc78ed75 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -27,13 +27,18 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk MODULES= \ ct_test_support \ ct_test_support_eh \ + ct_userconfig_callback \ ct_smoke_test_SUITE \ ct_event_handler_SUITE \ ct_groups_test_1_SUITE \ ct_groups_test_2_SUITE \ + ct_testspec_1_SUITE \ ct_skip_SUITE \ ct_error_SUITE \ - ct_test_server_if_1_SUITE + ct_test_server_if_1_SUITE \ + ct_config_SUITE \ + ct_master_SUITE \ + ct_misc_1_SUITE ERL_FILES= $(MODULES:%=%.erl) @@ -64,15 +69,15 @@ EBIN = . #.PHONY: make_emakefile #make_emakefile: -# $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) \ -# '*_SUITE_make' > $(EMAKEFILE) # $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES)\ -# >> $(EMAKEFILE) +# > $(EMAKEFILE) tests debug opt: + erl $(ERL_MAKE_FLAGS) -make clean: rm -f $(TARGET_FILES) +# rm -f $(EMAKEFILE) rm -f core docs: @@ -87,7 +92,7 @@ release_spec: opt release_tests_spec: $(INSTALL_DIR) $(RELSYSDIR) $(INSTALL_DATA) $(ERL_FILES) $(COVERFILE) $(RELSYSDIR) - $(INSTALL_PROGRAM) common_test.spec $(RELSYSDIR) + $(INSTALL_DATA) common_test.spec $(RELSYSDIR) chmod -f -R u+w $(RELSYSDIR) @tar cf - *_SUITE_data | (cd $(RELSYSDIR); tar xf -) diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl new file mode 100644 index 0000000000..72ff781f82 --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -0,0 +1,255 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_config_SUITE +%%% +%%% Description: +%%% Test configuration handling in Common Test suites. +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_config_SUITE). + +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + PathDir = filename:join(DataDir, "config/test"), + Config1 = ct_test_support:init_per_suite([{path_dirs,[PathDir]} | Config]), + PrivDir = ?config(priv_dir, Config1), + ConfigDir = filename:join(PrivDir, "config"), + ok = file:make_dir(ConfigDir), + [{config_dir,ConfigDir} | Config1]. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +all(doc) -> + [""]; + +all(suite) -> + [ + require, + userconfig_static, + userconfig_dynamic, + testspec_legacy, + testspec_static, + testspec_dynamic + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- +require(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + run_test(config_static_SUITE, + Config, + {config, filename:join(DataDir, "config/config.txt")}, + ["config_static_SUITE"]). + +userconfig_static(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + run_test(config_static_SUITE, + Config, + {userconfig, {ct_config_xml, filename:join(DataDir, "config/config.xml")}}, + ["config_static_SUITE"]). + +userconfig_dynamic(Config) when is_list(Config) -> + run_test(config_dynamic_SUITE, + Config, + {userconfig, {config_driver, "config_server"}}, + ["config_dynamic_SUITE"]). + +testspec_legacy(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + ConfigDir = ?config(config_dir, Config), + make_spec(DataDir, ConfigDir, + "spec_legacy.spec", + [config_static_SUITE], + [{config, filename:join(DataDir, "config/config.txt")}]), + run_test(config_static_SUITE, + Config, + {spec, filename:join(ConfigDir, "spec_legacy.spec")}, + []), + file:delete(filename:join(ConfigDir, "spec_legacy.spec")). + +testspec_static(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + ConfigDir = ?config(config_dir, Config), + make_spec(DataDir, ConfigDir, + "spec_static.spec", + [config_static_SUITE], + [{userconfig, {ct_config_xml, filename:join(DataDir, "config/config.xml")}}]), + run_test(config_static_SUITE, + Config, + {spec, filename:join(ConfigDir, "spec_static.spec")}, + []), + file:delete(filename:join(ConfigDir, "spec_static.spec")). + +testspec_dynamic(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + ConfigDir = ?config(config_dir, Config), + make_spec(DataDir, ConfigDir, "spec_dynamic.spec", + [config_dynamic_SUITE], + [{userconfig, {config_driver, "config_server"}}]), + run_test(config_dynamic_SUITE, + Config, + {spec, filename:join(ConfigDir, "spec_dynamic.spec")}, + []), + file:delete(filename:join(ConfigDir, "spec_dynamic.spec")). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +make_spec(DataDir, ConfigDir, Filename, Suites, Config)-> + {ok, Fd} = file:open(filename:join(ConfigDir, Filename), [write]), + ok = file:write(Fd, + io_lib:format("{suites, \"~sconfig/test/\", ~p}.~n", [DataDir, Suites])), + lists:foreach(fun(C)-> ok=file:write(Fd, io_lib:format("~p.~n", [C])) end, Config), + ok = file:close(Fd). + +run_test(Name, Config, CTConfig, SuiteNames)-> + DataDir = ?config(data_dir, Config), + Joiner = fun(Suite) -> filename:join(DataDir, "config/test/"++Suite) end, + Suites = lists:map(Joiner, SuiteNames), + {Opts,ERPid} = setup_env({suite,Suites}, Config, CTConfig), + ok = ct_test_support:run(Opts, Config), + TestEvents = ct_test_support:get_events(ERPid, Config), + ct_test_support:log_events(Name, + reformat_events(TestEvents, ?eh), + ?config(config_dir, Config)), + ExpEvents = events_to_check(Name), + ok = ct_test_support:verify_events(ExpEvents, TestEvents, Config). + +setup_env(Test, Config, CTConfig) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}, CTConfig], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat_events(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + expected_events(Test) ++ events_to_check(Test, N-1). + +expected_events(config_static_SUITE)-> +[ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,8}}, + {?eh,tc_start,{config_static_SUITE,init_per_suite}}, + {?eh,tc_done,{config_static_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_simple}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_simple,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_nested}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_nested,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_default_suitewide}}, + {?eh,tc_done,{config_static_SUITE,test_default_suitewide,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use1}}, + {?eh,tc_done, + {config_static_SUITE,test_config_name_already_in_use1,{skipped,{config_name_already_in_use,[x1]}}}}, + {?eh,test_stats,{3,0,{1,0}}}, + {?eh,tc_start,{config_static_SUITE,test_default_tclocal}}, + {?eh,tc_done,{config_static_SUITE,test_default_tclocal,ok}}, + {?eh,test_stats,{4,0,{1,0}}}, + {?eh,tc_start,{config_static_SUITE,test_config_name_already_in_use2}}, + {?eh,tc_done, + {config_static_SUITE,test_config_name_already_in_use2, + {skipped,{config_name_already_in_use,[x1,alias]}}}}, + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,test_alias_tclocal}}, + {?eh,tc_done,{config_static_SUITE,test_alias_tclocal,ok}}, + {?eh,test_stats,{5,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,test_get_config_undefined}}, + {?eh,tc_done,{config_static_SUITE,test_get_config_undefined,ok}}, + {?eh,test_stats,{6,0,{2,0}}}, + {?eh,tc_start,{config_static_SUITE,end_per_suite}}, + {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} +]; + +expected_events(config_dynamic_SUITE)-> +[ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,5}}, + {?eh,tc_start,{config_dynamic_SUITE,init_per_suite}}, + {?eh,tc_done,{config_dynamic_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{config_dynamic_SUITE,test_get_known_variable}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_get_known_variable,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,test_localtime_update}}, + {?eh,tc_done,{config_dynamic_SUITE,test_localtime_update,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,test_server_pid}}, + {?eh,tc_done,{config_dynamic_SUITE,test_server_pid,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start, + {config_dynamic_SUITE,test_disappearable_variable}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_disappearable_variable,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {config_dynamic_SUITE,test_disappearable_variable_alias}}, + {?eh,tc_done, + {config_dynamic_SUITE,test_disappearable_variable_alias,ok}}, + {?eh,test_stats,{5,0,{0,0}}}, + {?eh,tc_start,{config_dynamic_SUITE,end_per_suite}}, + {?eh,tc_done,{config_dynamic_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} +]. diff --git a/lib/common_test/test/ct_config_SUITE_data/config/config.txt b/lib/common_test/test/ct_config_SUITE_data/config/config.txt new file mode 100644 index 0000000000..fcbffcd7f3 --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE_data/config/config.txt @@ -0,0 +1,31 @@ +{x, suite}. +{gen_cfg, + [ + {a,a_value}, + {b,b_value} + ]}. +{gen_cfg2, + [ + {c, "Hello, world!"}, + {d, atom_value}, + {e, {tuple,1,"third value",[]}}, + {f, []}, + {g, [1,atom,"string",13.6,{1,2,3}]} + ]}. +{gen_cfg3, + [ + {h, + [ + {i, third1}, + {j, "Third2"}, + {k, 'THIRD3'} + ]}, + {l, + [ + {m, + [ + {n, "N"}, + {o, 'O'} + ]} + ]} + ]}. diff --git a/lib/common_test/test/ct_config_SUITE_data/config/config.xml b/lib/common_test/test/ct_config_SUITE_data/config/config.xml new file mode 100644 index 0000000000..0a3e5f2e31 --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE_data/config/config.xml @@ -0,0 +1,27 @@ +<config> + <x>suite</x> + <gen_cfg> + <a>a_value</a> + <b>b_value</b> + </gen_cfg> + <gen_cfg2> + <c>"Hello, world!"</c> + <d>atom_value</d> + <e>{tuple,1,"third value",[]}</e> + <f>[]</f> + <g>[1,atom,"string",13.6,{1,2,3}]</g> + </gen_cfg2> + <gen_cfg3> + <h> + <i>third1</i> + <j>"Third2"</j> + <k>'THIRD3'</k> + </h> + <l> + <m> + <n>"N"</n> + <o>'O'</o> + </m> + </l> + </gen_cfg3> +</config> diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl new file mode 100644 index 0000000000..d93faf6ec6 --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl @@ -0,0 +1,46 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_config_SUITE +%%% +%%% Description: +%%% Config driver used in the CT's tests (config_2_SUITE) +%%%------------------------------------------------------------------- +-module(config_driver). +-export([read_config/1, check_parameter/1]). + +read_config(ServerName)-> + ServerModule = list_to_atom(ServerName), + ServerModule:start(), + ServerModule:get_config(). + +check_parameter(ServerName)-> + ServerModule = list_to_atom(ServerName), + case code:is_loaded(ServerModule) of + {file, _}-> + {ok, {config, ServerName}}; + false-> + case code:load_file(ServerModule) of + {module, ServerModule}-> + {ok, {config, ServerName}}; + {error, nofile}-> + {error, {wrong_config, "File not found: " ++ ServerName ++ ".beam"}} + end + end. diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_dynamic_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_dynamic_SUITE.erl new file mode 100644 index 0000000000..8ee12a2e4d --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_dynamic_SUITE.erl @@ -0,0 +1,145 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: config_dynamic_SUITE +%%% +%%% Description: +%%% Test suite for common_test which tests the userconfig functionality +%%%------------------------------------------------------------------- +-module(config_dynamic_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%% This suite will be run with dynamic userconfig +%% test_driver.erl is compliant to ct_config_* callback +%% and test_server is simple server for getting runtime-changing data +%% which will return the list with the following variables: +%% localtime = the erlang:localtime() result in list [{date, Date}, {time, Time}] +%% node = erlang:node() - can be compared in the testcase +%% now = erlang:now() - easier to compare than localtime() +%% config_server_pid - pid of the config server, should NOT change! +%% config_server_vsn - .19 +%% config_server_iteration - a number of iteration config_server's loop done +%% disappearable_variable - hereAmI - will be absent on even iterations + +suite() -> + [ + {timetrap, {seconds,10}} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_) -> + ok. + +all() -> [test_get_known_variable, test_localtime_update, + test_server_pid, test_disappearable_variable, + test_disappearable_variable_alias]. + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _) -> + ok. + +% test that usual config works +test_get_known_variable(_)-> + Node = erlang:node(), + 0.19 = ct:get_config(config_server_vsn), + Node = ct:get_config(node), + ok. + +% localtime will be updated in 5 seconds, check that +test_localtime_update(_)-> + Seconds = 5, + LT1 = ct:get_config(localtime), + timer:sleep(Seconds*1000), + LT2 = ct:reload_config(localtime), + case is_diff_ok(LT1, LT2, Seconds) of + {false, Actual, Exp}-> + ct:fail(io_lib:format("Time difference ~p is not ok, expected ~p", [Actual, Exp])); + true-> + ok + end. + +% server pid should not change +test_server_pid()-> + [{require, cfvsn, config_server_vsn}]. +test_server_pid(_)-> + Pid = ct:get_config(config_server_pid), + Pid = ct:reload_config(config_server_pid), + Vsn = ct:get_config(config_server_vsn), + % aliases remain after config reloading + Vsn = ct:get_config(cfvsn), + ok. + +% test that variables may disappear from the config_2_SUITE +test_disappearable_variable(_)-> + % ask CT for config_server_iteration variable + Iter = ct:reload_config(config_server_iteration), + % here we should reload this variable in case it's odd + if Iter rem 2 == 1-> + Iter2 = ct:reload_config(config_server_iteration), + Iter2 = Iter+1; + true->ok + end, + % now disappearable_variable should be in place + hereAmI = ct:get_config(disappearable_variable), + % and now it should disappear + undefined = ct:reload_config(disappearable_variable). + +% alias of disappearable_variable should disappear too +test_disappearable_variable_alias(_)-> + % the same rules apply for this testcase as for previous one + Iter = ct:reload_config(config_server_iteration), + Iter2 = if + Iter rem 2 == 1 -> + NewIter = ct:reload_config(config_server_iteration), + NewIter = Iter+1; + true-> + Iter + end, + ct:require(diav, disappearable_variable), + hereAmI = ct:get_config(disappearable_variable), + hereAmI = ct:get_config(diav), + ct:reload_config(disappearable_variable), + undefined = ct:get_config(disappearable_variable), + % after reloading, it's even again + Iter3=ct:get_config(config_server_iteration), + Iter3 = Iter2+1, + % and alias does not exist + undefined = ct:get_config(diav). + +my_dt_to_datetime([{date, D},{time, T}])-> + {D, T}. + +is_diff_ok(DT1, DT2, Seconds)-> + GS1 = calendar:datetime_to_gregorian_seconds(my_dt_to_datetime(DT1)), + GS2 = calendar:datetime_to_gregorian_seconds(my_dt_to_datetime(DT2)), + if + GS2-GS1 > Seconds+Seconds/2; + GS2-GS1 < Seconds-Seconds/2-> + {false, GS2-GS1, Seconds}; + true-> + true + end. diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_server.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_server.erl new file mode 100644 index 0000000000..8463fea645 --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_server.erl @@ -0,0 +1,93 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_config_SUITE +%%% +%%% Description: +%%% Config server used in the CT's tests (config_2_SUITE) +%%%------------------------------------------------------------------- +-module(config_server). +-export([start/0, stop/0, loop/1, init/1, get_config/0]). + +-define(REGISTERED_NAME, ct_test_config_server). +-define(vsn, 0.19). + +start()-> + case whereis(?REGISTERED_NAME) of + undefined-> + spawn(?MODULE, init, [?REGISTERED_NAME]), + wait(); + _Pid-> + ok + end, + ?REGISTERED_NAME. + +init(Name)-> + register(Name, self()), + loop(0). + +get_config()-> + call(self(), get_config). + +stop()-> + call(self(), stop). + +call(Client, Request)-> + case whereis(?REGISTERED_NAME) of + undefined-> + {error, not_started, Request}; + Pid-> + Pid ! {Client, Request}, + receive + Reply-> + {ok, Reply} + after 4000-> + {error, timeout, Request} + end + end. + +loop(Iteration)-> + receive + {Pid, stop}-> + Pid ! ok; + {Pid, get_config}-> + {D,T} = erlang:localtime(), + Config = + [{localtime, [{date, D}, {time, T}]}, + {node, erlang:node()}, + {config_server_iteration, Iteration}, + {now, erlang:now()}, + {config_server_pid, self()}, + {config_server_vsn, ?vsn}], + Config2 = if Iteration rem 2 == 0-> + Config ++ [{disappearable_variable, hereAmI}]; + true-> Config + end, + Pid ! Config2, + ?MODULE:loop(Iteration+1) + end. + +wait()-> + case whereis(?REGISTERED_NAME) of + undefined-> + wait(); + _Pid-> + ok + end. diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl new file mode 100644 index 0000000000..8751a2e8f3 --- /dev/null +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl @@ -0,0 +1,123 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: config_static_SUITE +%%% +%%% Description: +%%% Test suite for common_test which tests the get_config and require +%%% functionality +%%%------------------------------------------------------------------- +-module(config_static_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +% The config contains variables: +% x - atom +% gen_cfg - list with two key-values tagged with a and b +% gen_cfg2 - list of five key-values tagged with c, d, e, f and g +% gen_cfg3 - list of two complex key-values taggen with: +% h: three elements inside - i, j and k +% l: m inside, contains n and o + +suite() -> + [ + {timetrap, {seconds,10}}, + %% x1 doesn't exist in cfg-file! + {require, x1, x}, + {require, gen_cfg3}, + {require, alias, gen_cfg}, + %% x1 default value + {x1, {x,suite}} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_) -> + ok. + +all() -> [test_get_config_simple, test_get_config_nested, test_default_suitewide, + test_config_name_already_in_use1, test_default_tclocal, + test_config_name_already_in_use2, test_alias_tclocal, + test_get_config_undefined]. + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _) -> + ok. + +%% test getting a simple value +test_get_config_simple(_)-> + suite = ct:get_config(x), + ok. + +%% test getting a nested value +test_get_config_nested(_)-> + a_value = ct:get_config({gen_cfg, a}), + ok. + +%% test suite-wide default value +test_default_suitewide(_)-> + suite = ct:get_config(x1), + ok. + +%% should get skipped +test_config_name_already_in_use1() -> + [{timetrap, {seconds,2}}, + {require, x1, x}, + {x1, {x,test2}}]. +test_config_name_already_in_use1(_) -> + ct:fail("Test should've been skipped, you shouldn't see this!"), + ok. + +%% test defaults in a testcase +test_default_tclocal() -> + [{timetrap, {seconds,3}}, + {require, y1, y}, + {y1, {y,test3}}]. +test_default_tclocal(_) -> + test3 = ct:get_config(y1), + ok. + +%% should get skipped +test_config_name_already_in_use2() -> + [{require,alias,something}, + {alias,{something,else}}, + {require, x1, x}, + {x1, {x,test4}}]. +test_config_name_already_in_use2(_) -> + ct:fail("Test should've been skipped, you shouldn't see this!"), + ok. + +%% test aliases +test_alias_tclocal() -> + [{require,newalias,gen_cfg}]. +test_alias_tclocal(_) -> + A = [{a,a_value},{b,b_value}] = ct:get_config(newalias), + A = ct:get_config(gen_cfg), + ok. + +%% test for getting undefined variables +test_get_config_undefined(_) -> + undefined = ct:get_config(y1), + ok. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index be75d768fc..2fa031b884 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -63,7 +63,10 @@ all(suite) -> [ cfg_error, lib_error, - no_compile + no_compile, + timetrap_end_conf, + timetrap_normal, + timetrap_extended ]. @@ -84,17 +87,22 @@ cfg_error(Config) when is_list(Config) -> Join(DataDir, "cfg_error_6_SUITE"), Join(DataDir, "cfg_error_7_SUITE"), Join(DataDir, "cfg_error_8_SUITE"), - Join(DataDir, "cfg_error_9_SUITE") + Join(DataDir, "cfg_error_9_SUITE"), + Join(DataDir, "cfg_error_10_SUITE"), + Join(DataDir, "cfg_error_11_SUITE"), + Join(DataDir, "cfg_error_12_SUITE"), + Join(DataDir, "cfg_error_13_SUITE"), + Join(DataDir, "cfg_error_14_SUITE") ], - {Opts,ERPid} = setup({suite,Suites}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(cfg_error, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(cfg_error), + TestEvents = events_to_check(cfg_error), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -104,15 +112,15 @@ lib_error(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, Suites = [Join(DataDir, "lib_error_1_SUITE")], - {Opts,ERPid} = setup({suite,Suites}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(lib_error, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(lib_error), + TestEvents = events_to_check(lib_error), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -122,17 +130,75 @@ no_compile(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, Suites = [Join(DataDir, "no_compile_SUITE")], - {Opts,ERPid} = setup({suite,Suites}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(no_compile, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(no_compile), + TestEvents = events_to_check(no_compile), ok = ct_test_support:verify_events(TestEvents, Events, Config). +%%%----------------------------------------------------------------- +%%% +timetrap_end_conf(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_1_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_end_conf, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(timetrap_end_conf), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% +timetrap_normal(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suite = Join(DataDir, "timetrap_2_SUITE"), + {Opts,ERPid} = setup([{suite,Suite}, + {userconfig,{ct_userconfig_callback, + "multiply 1 scale false"}}], + Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_normal, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(timetrap_normal), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% +timetrap_extended(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suite = Join(DataDir, "timetrap_2_SUITE"), + {Opts,ERPid} = setup([{suite,Suite}, + {multiply_timetraps,2}, + {scale_timetraps,false}, + {userconfig,{ct_userconfig_callback, + "multiply 2 scale false"}}], + Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_extended, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(timetrap_extended), + ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -142,7 +208,7 @@ setup(Test, Config) -> Opts0 = ct_test_support:get_opts(Config), Level = ?config(trace_level, Config), EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], - Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], ERPid = ct_test_support:start_event_receiver(Config), {Opts,ERPid}. @@ -154,11 +220,20 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + test_events(cfg_error) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{9,9,33}}, + {?eh,start_info,{14,14,42}}, {?eh,tc_start,{cfg_error_1_SUITE,init_per_suite}}, {?eh,tc_done, @@ -470,6 +545,59 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_9_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_9_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{cfg_error_10_SUITE,init_per_suite}}, + {?eh,tc_done,{cfg_error_10_SUITE,init_per_suite, + {failed,{error,fail_init_per_suite}}}}, + {?eh,tc_auto_skip,{cfg_error_10_SUITE,tc1, + {failed,{cfg_error_10_SUITE,init_per_suite, + {failed,fail_init_per_suite}}}}}, + {?eh,test_stats,{12,3,{0,19}}}, + {?eh,tc_auto_skip,{cfg_error_10_SUITE,end_per_suite, + {failed,{cfg_error_10_SUITE,init_per_suite, + {failed,fail_init_per_suite}}}}}, + {?eh,tc_start,{cfg_error_11_SUITE,init_per_suite}}, + {?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, + {?eh,tc_done,{cfg_error_11_SUITE,tc1, + {skipped,{config_name_already_in_use,[dummy0]}}}}, + {?eh,test_stats,{12,3,{1,19}}}, + {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, + {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, + {?eh,test_stats,{13,3,{1,19}}}, + {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, + {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc1,{failed,{timetrap_timeout,500}}}}, + {?eh,test_stats,{13,4,{1,19}}}, + {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, + {cfg_error_12_SUITE,end_per_testcase, + {timetrap_timeout,500}}}}}, + {?eh,test_stats,{14,4,{1,19}}}, + {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, + {?eh,test_stats,{15,4,{1,19}}}, + {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, + {cfg_error_12_SUITE,end_per_testcase, + {timetrap_timeout,500}}}}}, + {?eh,test_stats,{16,4,{1,19}}}, + {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, + {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, + {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, + {?eh,test_stats,{17,4,{1,19}}}, + {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, + {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, + {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, + {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, + {?eh,test_stats,{18,4,{1,19}}}, + {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, + {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, + {comment, + "should succeed since ct_fw cancels timetrap in end_tc"}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; @@ -555,4 +683,91 @@ test_events(lib_error) -> ]; test_events(no_compile) -> - []. + []; + +test_events(timetrap_end_conf) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,6}}, + {?eh,tc_start,{timetrap_1_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_1_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_1_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_1_SUITE,tc1,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_1_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_1_SUITE,tc2,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_1_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_1_SUITE,tc3,{failed,{testcase_aborted,testing_end_conf}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_1_SUITE,tc4}}, + {?eh,tc_done, + {timetrap_1_SUITE,tc4,{failed,{testcase_aborted,testing_end_conf}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{timetrap_1_SUITE,tc5}}, + {?eh,tc_done, + {timetrap_1_SUITE,tc5,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_1_SUITE,tc6}}, + {?eh,tc_done, + {timetrap_1_SUITE,tc6,{failed,{testcase_aborted,testing_end_conf}}}}, + {?eh,test_stats,{0,6,{0,0}}}, + {?eh,tc_start,{timetrap_1_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_1_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(timetrap_normal) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,3}}, + {?eh,tc_start,{timetrap_2_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_2_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_2_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_2_SUITE,tc0,{failed,{timetrap_timeout,3000}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_2_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_2_SUITE,tc1,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_2_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_2_SUITE,tc2,{failed,{timetrap_timeout,500}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_2_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_2_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(timetrap_extended) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,3}}, + {?eh,tc_start,{timetrap_2_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_2_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_2_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_2_SUITE,tc0,{failed,{timetrap_timeout,6000}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_2_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_2_SUITE,tc1,{failed,{timetrap_timeout,2000}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_2_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_2_SUITE,tc2,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_2_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_2_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_10_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_10_SUITE.erl new file mode 100644 index 0000000000..9f9a90372b --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_10_SUITE.erl @@ -0,0 +1,123 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(cfg_error_10_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,2}}, + {require, dummy0}, {default_config, dummy0, "suite/0"}, + {require, dummy1}, {default_config, dummy1, "suite/0"}, + {require, dummy2}, {default_config, dummy2, "suite/0"}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite() -> + put('$test_server_framework_test', + fun(init_tc, _Default) -> {fail,fail_init_per_suite}; + (_, Default) -> Default + end), + [{require, dummy3}, {default_config, dummy3, "init_per_suite/0"}, + {timetrap,3000}]. + +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + done. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc1]. + +tc1(_) -> + exit(should_never_run). diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl new file mode 100644 index 0000000000..ce94533110 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_11_SUITE.erl @@ -0,0 +1,134 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(cfg_error_11_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,2}}, + {require, dummy0}, {default_config, dummy0, "suite/0"}, + {require, dummy1}, {default_config, dummy1, "suite/0"}, + {require, dummy2}, {default_config, dummy2, "suite/0"}]. + + + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite() -> + put('$test_server_framework_test', + fun(end_tc, _Default) -> {fail,fail_end_per_suite}; + (_, Default) -> Default + end), + [{require, dummy3}, {default_config, dummy3, "end_per_suite/0"}, + {timetrap,3000}]. + +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + done. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc1, tc2]. + +tc1() -> + [{require, dummy0}, {default_config, dummy0, "tc1"}]. + +tc1(_) -> + dummy. + +tc2() -> + [{timetrap,1}]. + +tc2(_) -> + dummy. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_12_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_12_SUITE.erl new file mode 100644 index 0000000000..806d3caf72 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_12_SUITE.erl @@ -0,0 +1,88 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(cfg_error_12_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(tc2, _Config) -> + timer:sleep(2000), + exit(this_should_not_be_printed); +end_per_testcase(tc4, _Config) -> + timer:sleep(2000), + exit(this_should_not_be_printed); +end_per_testcase(_, _) -> + ok. + +all() -> + [tc1, tc2, tc3, tc4]. + +%%%----------------------------------------------------------------- +tc1() -> + put('$test_server_framework_test', + fun(init_tc, _Default) -> + ct:pal("init_tc(~p): Night time...",[self()]), + timer:sleep(2000), + ct:pal("init_tc(~p): Day time!",[self()]), + exit(this_should_not_be_printed); + (_, Default) -> Default + end), + [{timetrap,500}]. + +tc1(_) -> + exit(this_should_not_be_printed). + +%%%----------------------------------------------------------------- +tc2() -> + [{timetrap,500}]. + +tc2(_) -> + ok. + +%%%----------------------------------------------------------------- +tc3() -> + [{timetrap,500}]. + +tc3(_) -> + put('$test_server_framework_test', + fun(end_tc, _Default) -> + ct:pal("end_tc(~p): Night time...",[self()]), + timer:sleep(1000), + ct:pal("end_tc(~p): Day time!",[self()]); + (_, Default) -> Default + end), + {comment,"should succeed since ct_fw cancels timetrap in end_tc"}. + +%%%----------------------------------------------------------------- +tc4() -> + put('$test_server_framework_test', + fun(end_tc, _Default) -> + ct:pal("end_tc(~p): Night time...",[self()]), + timer:sleep(1000), + ct:pal("end_tc(~p): Day time!",[self()]); + (_, Default) -> Default + end), + [{timetrap,500}]. + +tc4(_) -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_13_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_13_SUITE.erl new file mode 100644 index 0000000000..c8a3c1d15e --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_13_SUITE.erl @@ -0,0 +1,47 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(cfg_error_13_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +init_per_suite() -> + put('$test_server_framework_test', + fun(end_tc, _Default) -> + ct:pal("end_tc(~p): Night time...",[self()]), + timer:sleep(1000), + ct:pal("end_tc(~p): Day time!",[self()]); + (_, Default) -> Default + end), + [{timetrap,500}]. + +init_per_suite(Config) -> + ct:comment("should succeed since ct_fw cancels timetrap in end_tc"), + Config. + +end_per_suite(_) -> + ok. + +all() -> + [tc1]. + +%%%----------------------------------------------------------------- +tc1(_) -> + dummy. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_14_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_14_SUITE.erl new file mode 100644 index 0000000000..960d0f61b0 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_14_SUITE.erl @@ -0,0 +1,46 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(cfg_error_14_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +init_per_suite(Config) -> + Config. + +end_per_suite() -> + put('$test_server_framework_test', + fun(end_tc, _Default) -> + ct:pal("end_tc(~p): Night time...",[self()]), + timer:sleep(1000), + ct:pal("end_tc(~p): Day time!",[self()]); + (_, Default) -> Default + end), + [{timetrap,500}]. + +end_per_suite(_Config) -> + {comment,"should succeed since ct_fw cancels timetrap in end_tc"}. + +all() -> + [tc1]. + +%%%----------------------------------------------------------------- +tc1(_) -> + dummy. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_3_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_3_SUITE.erl index bf01bb52d9..08c57887ef 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_3_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_3_SUITE.erl @@ -37,7 +37,8 @@ suite() -> %%-------------------------------------------------------------------- init_per_suite(Config) -> timer:sleep(5000), - Config. + exit(shouldnt_happen). +% Config. %%-------------------------------------------------------------------- %% Function: end_per_suite(Config0) -> void() | {save_config,Config1} diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl new file mode 100644 index 0000000000..cb3109349b --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_1_SUITE.erl @@ -0,0 +1,194 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_1_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,1}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + TabPid = spawn(fun() -> + ets:new(?MODULE, [named_table, set, public]), + ets:insert(?MODULE, {last_case,ok}), + receive _ -> ok end + end), + [{tab,TabPid} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + exit(?config(tab, Config), kill), + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(TC, Config) -> + {_,_} = process_info(?config(tab, Config), priority), + [{_,ok}] = ets:lookup(?MODULE, last_case), + ets:insert(?MODULE, {last_case,fail}), + init_per_testcase1(TC, Config). + +init_per_testcase1(tc1, Config) -> + [{tc,tc1}|Config]; + +init_per_testcase1(tc2, Config) -> + [{tc,tc2}|Config]; + +init_per_testcase1(tc3, Config) -> + [{tc,tc3}|Config]; + +init_per_testcase1(tc4, Config) -> + [{tc,tc4},{default_timeout,5000}|Config]; + +init_per_testcase1(tc5, Config) -> + [{tc,tc5}|Config]; + +init_per_testcase1(tc6, Config) -> + [{tc,tc6}|Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(TC, Config) -> + {_,_} = process_info(?config(tab, Config), priority), + [{_,fail}] = ets:lookup(?MODULE, last_case), + ets:insert(?MODULE, {last_case,ok}), + end_per_testcase1(TC, Config). + +end_per_testcase1(tc1, Config) -> + ct:pal("end_per_testcase(tc1): ~p", [Config]), + tc1 = ?config(tc, Config), + {failed,timetrap_timeout} = ?config(tc_status, Config), + ok; + +end_per_testcase1(tc2, Config) -> + ct:pal("end_per_testcase(tc2): ~p", [Config]), + tc2 = ?config(tc, Config), + {failed,timetrap_timeout} = ?config(tc_status, Config), + timer:sleep(2000); + +end_per_testcase1(tc3, Config) -> + ct:pal("end_per_testcase(tc3): ~p", [Config]), + tc3 = ?config(tc, Config), + {failed,{testcase_aborted,testing_end_conf}} = ?config(tc_status, Config), + ok; + +end_per_testcase1(tc4, Config) -> + ct:pal("end_per_testcase(tc4): ~p", [Config]), + tc4 = ?config(tc, Config), + {failed,{testcase_aborted,testing_end_conf}} = ?config(tc_status, Config), + timer:sleep(2000); + +end_per_testcase1(tc5, Config) -> + ct:pal("end_per_testcase(tc5): ~p", [Config]), + tc5 = ?config(tc, Config), + exit(end_per_tc_fail_after_timeout); + +end_per_testcase1(tc6, Config) -> + ct:pal("end_per_testcase(tc6): ~p", [Config]), + tc6 = ?config(tc, Config), + exit(end_per_tc_fail_after_abort). + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc1, tc2, tc3, tc4, tc5, tc6]. + +tc1(_) -> + timer:sleep(2000). + +tc2(_) -> + timer:sleep(2000). + +tc3(_) -> + spawn(ct, abort_current_testcase, [testing_end_conf]), + timer:sleep(2000). + +tc4(_) -> + spawn(ct, abort_current_testcase, [testing_end_conf]), + timer:sleep(2000). + +tc5(_) -> + timer:sleep(2000). + +tc6(_) -> + spawn(ct, abort_current_testcase, [testing_end_conf]), + timer:sleep(2000). diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_2_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_2_SUITE.erl new file mode 100644 index 0000000000..99bb400137 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_2_SUITE.erl @@ -0,0 +1,138 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_2_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,3}}, + {require,multiply}, + {require,scale}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(tc1, Config) -> + ct:timetrap({seconds,1}), + Config; + +init_per_testcase(tc3, Config) -> + ct:timetrap({seconds,1}), + Config; + +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2]. + +tc0(_) -> + N = list_to_integer(ct:get_config(multiply)), + ct:comment(io_lib:format("TO after ~w sec", [3*N])), + ct:sleep({seconds,5}), + ok. + +tc1(_) -> + N =list_to_integer( ct:get_config(multiply)), + ct:comment(io_lib:format("TO after ~w sec", [1*N])), + ct:sleep({seconds,5}), + ok. + +tc2(_) -> + N = list_to_integer(ct:get_config(multiply)), + ct:comment(io_lib:format("TO after ~w sec", [0.5*N])), + ct:timetrap(500), + ct:sleep(2000), + ok. diff --git a/lib/common_test/test/ct_event_handler_SUITE.erl b/lib/common_test/test/ct_event_handler_SUITE.erl index bafd32f937..00a4c4ded3 100644 --- a/lib/common_test/test/ct_event_handler_SUITE.erl +++ b/lib/common_test/test/ct_event_handler_SUITE.erl @@ -88,7 +88,7 @@ start_stop(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -110,8 +110,7 @@ start_stop(Config) when is_list(Config) -> {eh_A,test_done,{'DEF','STOP_TIME'}}, {eh_A,stop_logging,[]}], - ok = ct_test_support:verify_events(TestEvents, Events, Config), - {comment,"NOTE! Known problem with test_start event!"}. + ok = ct_test_support:verify_events(TestEvents++TestEvents, Events, Config). results(doc) -> @@ -135,7 +134,7 @@ results(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -163,7 +162,7 @@ results(Config) when is_list(Config) -> {eh_A,test_done,{'DEF','STOP_TIME'}}, {eh_A,stop_logging,[]}], - ok = ct_test_support:verify_events(TestEvents, Events, Config). + ok = ct_test_support:verify_events(TestEvents++TestEvents, Events, Config). %%%----------------------------------------------------------------- diff --git a/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl b/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl index 6e526f15a2..54cf3a22e7 100644 --- a/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl +++ b/lib/common_test/test/ct_event_handler_SUITE_data/eh_A.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2008-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -44,6 +44,19 @@ %% Description: Whenever a new event handler is added to an event manager, %% this function is called to initialize the event handler. %%-------------------------------------------------------------------- +init(String = [X|_]) when is_integer(X) -> + case erl_scan:string(String++".") of + {ok,Ts,_} -> + case erl_parse:parse_term(Ts) of + {ok,Args} -> + init(Args); + _ -> + init(String) + end; + _ -> + init(String) + end; + init(Args) -> S1 = case lists:keysearch(cbm, 1, Args) of diff --git a/lib/common_test/test/ct_groups_test_1_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE.erl index 1761b773f5..64d61fc104 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE.erl @@ -76,14 +76,14 @@ groups_suite_1(Config) when is_list(Config) -> Suite = filename:join(DataDir, "groups_1/test/groups_11_SUITE"), {Opts,ERPid} = setup({suite,Suite}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(groups_suite_1, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(groups_suite_1), + TestEvents = events_to_check(groups_suite_1), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -96,14 +96,14 @@ groups_suite_2(Config) when is_list(Config) -> Suite = filename:join(DataDir, "groups_1/test/groups_12_SUITE"), {Opts,ERPid} = setup({suite,Suite}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(groups_suite_2, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(groups_suite_2), + TestEvents = events_to_check(groups_suite_2), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -117,14 +117,14 @@ groups_suites_1(Config) when is_list(Config) -> filename:join(DataDir, "groups_1/test/groups_12_SUITE")], {Opts,ERPid} = setup({suite,Suites}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(groups_suites_1, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(groups_suites_1), + TestEvents = events_to_check(groups_suites_1), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -137,14 +137,14 @@ groups_dir_1(Config) when is_list(Config) -> Dir = filename:join(DataDir, "groups_1"), {Opts,ERPid} = setup({dir,Dir}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(groups_dir_1, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(groups_dir_1), + TestEvents = events_to_check(groups_dir_1), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -157,14 +157,14 @@ groups_dirs_1(Config) when is_list(Config) -> filename:join(DataDir, "groups_2")], {Opts,ERPid} = setup({dir,Dirs}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(groups_dirs_1, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(groups_dirs_1), + TestEvents = events_to_check(groups_dirs_1), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -188,6 +188,14 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). test_events(groups_suite_1) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, @@ -327,14 +335,14 @@ test_events(groups_suite_2) -> {?eh,tc_start,{groups_12_SUITE,testcase_2a}}, {?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}}, - [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, {?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3b}}, {?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}], + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}], [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}}, @@ -361,12 +369,8 @@ test_events(groups_suite_2) -> {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - - %% the done event could come in during the parallel subgroup - %% and we can't test that, yet... - %% {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, - %% {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, - + {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, @@ -525,14 +529,14 @@ test_events(groups_suites_1) -> {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_2a}}, {?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}}, - [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, {?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3b}}, {?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}], + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}], [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, @@ -555,12 +559,8 @@ test_events(groups_suites_1) -> {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - - %% the done event could come in during the parallel subgroup - %% and we can't test that, yet... - %% {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, - %% {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, - + {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,[sequence]}}}, @@ -715,14 +715,14 @@ test_events(groups_dir_1) -> {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_2a}}, {?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}}, - [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, {?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3b}}, {?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}], + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}], [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, @@ -745,12 +745,8 @@ test_events(groups_dir_1) -> {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - - %% the done event could come in during the parallel subgroup - %% and we can't test that, yet... - %% {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, - %% {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, - + {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,[sequence]}}}, @@ -906,14 +902,14 @@ test_events(groups_dirs_1) -> {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_2a}}, {?eh,tc_done,{groups_12_SUITE,testcase_2a,ok}}, - [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, {?eh,tc_done,{groups_12_SUITE,testcase_3a,ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3b}}, {?eh,tc_done,{groups_12_SUITE,testcase_3b,ok}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}}, - {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}], + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}], [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_3,[]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_3,[]},ok}}, {?eh,tc_start,{groups_12_SUITE,testcase_3a}}, @@ -936,12 +932,8 @@ test_events(groups_dirs_1) -> {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - - %% the done event could come in during the parallel subgroup - %% and we can't test that, yet... - %% {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, - %% {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, - + {?eh,tc_start,{groups_12_SUITE,testcase_5a}}, + {?eh,tc_done,{groups_12_SUITE,testcase_5a,ok}}, {parallel,[{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,[sequence]}}}, @@ -1138,17 +1130,17 @@ test_events(groups_dirs_1) -> {?eh,tc_start,{groups_22_SUITE,testcase_2a}}, {?eh,tc_done,{groups_22_SUITE,testcase_2a,ok}}, [{?eh,tc_start, - {groups_22_SUITE,{init_per_group,test_group_3,[{repeat,1}]}}}, + {groups_22_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}}, {?eh,tc_done, - {groups_22_SUITE,{init_per_group,test_group_3,[{repeat,1}]},ok}}, + {groups_22_SUITE,{init_per_group,test_group_3,[{repeat,2}]},ok}}, {?eh,tc_start,{groups_22_SUITE,testcase_3a}}, {?eh,tc_done,{groups_22_SUITE,testcase_3a,ok}}, {?eh,tc_start,{groups_22_SUITE,testcase_3b}}, {?eh,tc_done,{groups_22_SUITE,testcase_3b,ok}}, {?eh,tc_start, - {groups_22_SUITE,{end_per_group,test_group_3,[{repeat,1}]}}}, + {groups_22_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}, {?eh,tc_done, - {groups_22_SUITE,{end_per_group,test_group_3,[{repeat,1}]},ok}}], + {groups_22_SUITE,{end_per_group,test_group_3,[{repeat,2}]},ok}}], [{?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_3,[]}}}, {?eh,tc_done, @@ -1181,12 +1173,8 @@ test_events(groups_dirs_1) -> {groups_22_SUITE,{init_per_group,test_group_5,[parallel]}}}, {?eh,tc_done, {groups_22_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - - %% the done event could come in during the parallel subgroup - %% and we can't test that, yet... - %% {?eh,tc_start,{groups_22_SUITE,testcase_5a}}, - %% {?eh,tc_done,{groups_22_SUITE,testcase_5a,ok}}, - + {?eh,tc_start,{groups_22_SUITE,testcase_5a}}, + {?eh,tc_done,{groups_22_SUITE,testcase_5a,ok}}, {parallel, [{?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_6,[parallel]}}}, diff --git a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl index c6d50443d0..c69400e938 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl @@ -99,6 +99,7 @@ init_per_group(Group, Config) -> {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of true -> + ct:comment(Group), init = ?config(suite,Config), [{Group,Group} | Config]; false -> @@ -109,6 +110,7 @@ end_per_group(Group, Config) -> {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of true -> + ct:comment(Group), init = ?config(suite,Config), Group = ?config(Group,Config), ok; diff --git a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_12_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_12_SUITE.erl index b261ef581f..ec90ef95d1 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_12_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_12_SUITE.erl @@ -37,7 +37,7 @@ groups() -> {test_group_2, [parallel], [testcase_2a, - {test_group_3, [{repeat,1}], + {test_group_3, [{repeat,2}], [testcase_3a, testcase_3b]}, testcase_2b]}, @@ -102,8 +102,8 @@ init_per_group(Group, Config) -> io_lib:format("shuffled, ~w", [S]); {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; - {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1"; - {test_group_3,[{name,test_group_3}]} -> "repeat 0"; + {test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2"; + {test_group_3,[{name,test_group_3}]} -> "repeat 1"; {test_group_4,[{name,test_group_4}]} -> ok; {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; @@ -275,6 +275,10 @@ testcase_5a(Config) -> test_group_5 = ?config(test_group_5,Config), undefined = ?config(testcase_3,Config), testcase_5a = ?config(testcase_5a,Config), + %% increase chance the done event will come + %% during execution of subgroup (could be + %% tricky to handle) + timer:sleep(3), ok. testcase_5b() -> []. diff --git a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_2/test/groups_22_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_2/test/groups_22_SUITE.erl index 2e19cf6310..ec0adc5df0 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_2/test/groups_22_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_2/test/groups_22_SUITE.erl @@ -37,7 +37,7 @@ groups() -> {test_group_2, [parallel], [testcase_2a, - {test_group_3, [{repeat,1}], + {test_group_3, [{repeat,2}], [testcase_3a, testcase_3b]}, testcase_2b]}, @@ -102,8 +102,8 @@ init_per_group(Group, Config) -> io_lib:format("shuffled, ~w", [S]); {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; - {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1"; - {test_group_3,[{name,test_group_3}]} -> "repeat 0"; + {test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2"; + {test_group_3,[{name,test_group_3}]} -> "repeat 1"; {test_group_4,[{name,test_group_4}]} -> ok; {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index 5a60d855b7..c4371501b3 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -60,7 +60,7 @@ all(doc) -> ["Run smoke tests of Common Test."]; all(suite) -> - [missing_conf]. + [missing_conf, repeat_1]. %%-------------------------------------------------------------------- %% TEST CASES @@ -75,16 +75,35 @@ missing_conf(Config) when is_list(Config) -> Suite = filename:join(DataDir, "groups_1/missing_conf_SUITE"), {Opts,ERPid} = setup({suite,Suite}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(missing_conf_SUITE, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(missing_conf), + TestEvents = events_to_check(missing_conf), ok = ct_test_support:verify_events(TestEvents, Events, Config). - + +%%%----------------------------------------------------------------- +%%% + +repeat_1(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + Suite = filename:join(DataDir, "groups_1/repeat_1_SUITE"), + + {Opts,ERPid} = setup({suite,Suite}, Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(repeat_1, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(repeat_1), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -105,6 +124,127 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). test_events(missing_conf) -> - exit(must_handle_this). + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_framework,ct_init_per_group}}, + {?eh,tc_done,{ct_framework,ct_init_per_group,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{missing_conf_SUITE,tc1}}, + {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{missing_conf_SUITE,tc2}}, + {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{ct_framework,ct_end_per_group}}, + {?eh,tc_done,{ct_framework,ct_end_per_group,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_1) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,unknown}}, + {?eh,tc_start,{repeat_1_SUITE,init_per_suite}}, + {?eh,tc_done,{repeat_1_SUITE,init_per_suite,ok}}, + [{?eh,tc_start, + {repeat_1_SUITE,{init_per_group,test_group_1,[{repeat,2}]}}}, + {?eh,tc_done, + {repeat_1_SUITE,{init_per_group,test_group_1,[{repeat,2}]},ok}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_1a}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_1a,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_1b}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_1b,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start, + {repeat_1_SUITE,{end_per_group,test_group_1,[{repeat,2}]}}}, + {?eh,tc_done, + {repeat_1_SUITE,{end_per_group,test_group_1,[{repeat,2}]},ok}}], + [{?eh,tc_start, + {repeat_1_SUITE,{init_per_group,test_group_1,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE,{init_per_group,test_group_1,[]},ok}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_1a}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_1a,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_1b}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_1b,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start, + {repeat_1_SUITE,{end_per_group,test_group_1,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE,{end_per_group,test_group_1,[]},ok}}], + [{?eh,tc_start, + {repeat_1_SUITE,{init_per_group,test_group_2,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE,{init_per_group,test_group_2,[]},ok}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_2a}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_2a,ok}}, + {?eh,test_stats,{5,0,{0,0}}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_2b}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_2b,ok}}, + {?eh,test_stats,{6,0,{0,0}}}, + {?eh,tc_start, + {repeat_1_SUITE,{end_per_group,test_group_2,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE,{end_per_group,test_group_2,[]},ok}}], + [{?eh,tc_start, + {repeat_1_SUITE, + {init_per_group,test_group_3,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE, + {init_per_group,test_group_3,[]}, + ok}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_3a}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_3a,ok}}, + {?eh,test_stats,{7,0,{0,0}}}, + [{?eh,tc_start, + {repeat_1_SUITE, + {init_per_group,test_group_4,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE, + {init_per_group,test_group_4,[]}, + ok}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_4a}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_4a,ok}}, + {?eh,test_stats,{8,0,{0,0}}}, + {?eh,tc_start,{repeat_1_SUITE,testcase_4b}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_4b,ok}}, + {?eh,test_stats,{9,0,{0,0}}}, + {?eh,tc_start, + {repeat_1_SUITE, + {end_per_group,test_group_4,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE, + {end_per_group,test_group_4,[]}, + ok}}], + {?eh,tc_start,{repeat_1_SUITE,testcase_3b}}, + {?eh,tc_done,{repeat_1_SUITE,testcase_3b,ok}}, + {?eh,test_stats,{10,0,{0,0}}}, + {?eh,tc_start, + {repeat_1_SUITE, + {end_per_group,test_group_3,[]}}}, + {?eh,tc_done, + {repeat_1_SUITE, + {end_per_group,test_group_3,[]}, + ok}}], + {?eh,tc_start,{repeat_1_SUITE,end_per_suite}}, + {?eh,tc_done,{repeat_1_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/cfgs/groups_2.1.cfg b/lib/common_test/test/ct_groups_test_2_SUITE_data/cfgs/groups_2.1.cfg new file mode 100644 index 0000000000..4928505157 --- /dev/null +++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/cfgs/groups_2.1.cfg @@ -0,0 +1 @@ +{dummy_key, "dummy_data"}. diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_1/repeat_1_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_1/repeat_1_SUITE.erl new file mode 100644 index 0000000000..b4b9b03ca5 --- /dev/null +++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_1/repeat_1_SUITE.erl @@ -0,0 +1,105 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(repeat_1_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1, [{repeat,2}], [testcase_1a,testcase_1b]}, + {test_group_2, [{repeat,1}], [testcase_2a,testcase_2b]}, + + {test_group_3, [{repeat_until_all_fail,1}], + [testcase_3a, + {test_group_4, [{repeat_until_any_fail,1}], + [testcase_4a, testcase_4b]}, + testcase_3b]} + ]. + +all() -> + [ + {group, test_group_1}, + {group, test_group_2}, + {group, test_group_3} + ]. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + Group = proplists:get_value(name,?config(tc_group_properties,Config)), + ct:comment(Group), + Config. + +end_per_group(_Group, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1a(_) -> + ok. +testcase_1b(_) -> + ok. + +testcase_2a(_) -> + ok. +testcase_2b(_) -> + ok. + +testcase_3a(_) -> + ok. +testcase_3b(_) -> + ok. + +testcase_4a(_) -> + ok. +testcase_4b(_) -> + ok. diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_21_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_21_SUITE.erl new file mode 100644 index 0000000000..2533ac8e84 --- /dev/null +++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_21_SUITE.erl @@ -0,0 +1,281 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_21_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [testcase_1a,testcase_1b]}, + + {test_group_1b, [], [testcase_1a,testcase_1b]}, + + {test_group_2, [], [testcase_2a, + + {test_group_3, [], [testcase_3a, + testcase_3b]}, + testcase_2b]}, + + {test_group_4, [{test_group_5, [], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [{group, test_group_7}]}, + + {test_group_7, [testcase_7a,testcase_7b]} + ]. + +all() -> + [testcase_1, + {group, test_group_1a}, + {group, test_group_1b}, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1, + testcase_1a, testcase_1b, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + [{name,Group}] = ?config(tc_group_properties,Config), + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(io_lib:format("~w", [Group])), + init = ?config(suite,Config), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_22_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_22_SUITE.erl new file mode 100644 index 0000000000..cd517876df --- /dev/null +++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_22_SUITE.erl @@ -0,0 +1,314 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_22_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]}, + + {test_group_1b, [parallel], [testcase_1a,testcase_1b]}, + + {test_group_2, [parallel], [testcase_2a, + + {test_group_3, [{repeat,1}], + [testcase_3a, testcase_3b]}, + + testcase_2b]}, + + {test_group_4, [{test_group_5, [parallel], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [parallel], [{group, test_group_7}]}, + + {test_group_7, [sequence], [testcase_7a,testcase_7b]} + ]. + +all() -> + [{group, test_group_1a}, + {group, test_group_1b}, + testcase_1, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1a, testcase_1b, testcase_1c, + testcase_1, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + Cmt = + case {Group,?config(tc_group_properties,Config)} of + {test_group_1a,[{shuffle,S},{name,test_group_1a}]} -> + io_lib:format("shuffled, ~w", [S]); + {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; + {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; + {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1"; + {test_group_3,[{name,test_group_3}]} -> "repeat 0"; + {test_group_4,[{name,test_group_4}]} -> ok; + {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; + {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; + {test_group_7,[{name,test_group_7},sequence]} -> "sequence" + end, + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + ct:comment(io_lib:format("~w, ~s", [Group,Cmt])), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_1c() -> + []. +testcase_1c(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1b,Config), + testcase_1c = ?config(testcase_1c,Config), + ok. + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + %% increase chance the done event will come + %% during execution of subgroup (could be + %% tricky to handle) + timer:sleep(3), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl new file mode 100644 index 0000000000..87e2c3049a --- /dev/null +++ b/lib/common_test/test/ct_master_SUITE.erl @@ -0,0 +1,136 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_master_SUITE +%%% +%%% Description: +%%% Test ct_master. +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_master_SUITE). +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, [{master, true}|Config]). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +all(doc) -> + [""]; + +all(suite) -> + [ + ct_master_test + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- +ct_master_test(Config) when is_list(Config)-> + NodeCount = 5, + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + NodeNames = [list_to_atom("testnode_"++integer_to_list(N)) || + N <- lists:seq(1, NodeCount)], + FileName = filename:join(PrivDir, "ct_master_spec.spec"), + Suites = [master_SUITE], + TSFile = make_spec(DataDir, FileName, NodeNames, Suites, Config), + [{TSFile, ok}] = run_test(ct_master_test, FileName, Config), + ok. + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +make_spec(DataDir, FileName, NodeNames, Suites, Config)-> + {ok, HostName} = inet:gethostname(), + + N = lists:map(fun(NodeName)-> + {node, NodeName, list_to_atom(atom_to_list(NodeName)++"@"++HostName)} + end, + NodeNames), + + C = lists:map(fun(NodeName)-> + Rnd = random:uniform(2), + if Rnd == 1-> + {config, NodeName, filename:join(DataDir, "master/config.txt")}; + true-> + {userconfig, NodeName, {ct_config_xml, filename:join(DataDir, "master/config.xml")}} + end + end, + NodeNames), + + NS = lists:map(fun(NodeName)-> + {init, NodeName, [ + {node_start, [{startup_functions, []}, {monitor_master, false}]}, + {eval, {erlang, nodes, []}} + ] + } + end, + NodeNames), + + S = [{suites, NodeNames, filename:join(DataDir, "master"), Suites}], + + PrivDir = ?config(priv_dir, Config), + LD = lists:map(fun(NodeName)-> + {logdir, NodeName, get_log_dir(PrivDir, NodeName)} + end, + NodeNames) ++ [{logdir, master, PrivDir}], + + ct_test_support:write_testspec(N++C++S++LD++NS, FileName). + +get_log_dir(PrivDir, NodeName)-> + LogDir = filename:join(PrivDir, io_lib:format("slave.~p", [NodeName])), + file:make_dir(LogDir), + LogDir. + +run_test(_Name, FileName, Config)-> + [{FileName, ok}] = ct_test_support:run(ct_master, run, [FileName], Config). + +reformat_events(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +expected_events(_)-> +[]. diff --git a/lib/common_test/test/ct_master_SUITE_data/master/config.txt b/lib/common_test/test/ct_master_SUITE_data/master/config.txt new file mode 100644 index 0000000000..3baf9e392c --- /dev/null +++ b/lib/common_test/test/ct_master_SUITE_data/master/config.txt @@ -0,0 +1,2 @@ +{a, b}. +{c, d}. diff --git a/lib/common_test/test/ct_master_SUITE_data/master/config.xml b/lib/common_test/test/ct_master_SUITE_data/master/config.xml new file mode 100644 index 0000000000..c031f45f35 --- /dev/null +++ b/lib/common_test/test/ct_master_SUITE_data/master/config.xml @@ -0,0 +1,4 @@ +<config> + <a>b</a> + <c>d</c> +</config> diff --git a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl new file mode 100644 index 0000000000..e37ec3659c --- /dev/null +++ b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl @@ -0,0 +1,57 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: master_SUITE +%%% +%%% Description: +%%% Test suite for common_test which tests the ct_master functionality +%%%------------------------------------------------------------------- +-module(master_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_) -> + ok. + +all() -> [first_testcase, second_testcase, third_testcase]. + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _) -> + ok. + +first_testcase(_)-> + b = ct:get_config(a). + +second_testcase(_)-> + d = ct:get_config(c). + +third_testcase(_)-> + A = 4, + A = 2*2. diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl new file mode 100644 index 0000000000..eb6c6aa101 --- /dev/null +++ b/lib/common_test/test/ct_misc_1_SUITE.erl @@ -0,0 +1,165 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_misc_1_SUITE +%%% +%%% Description: +%%% Test misc things in Common Test suites. +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_misc_1_SUITE). + +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("test_server/include/test_server_line.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +all(doc) -> + [""]; + +all(suite) -> + [ + beam_me_up + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +beam_me_up(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + CTNode = ?config(ct_node, Config), + + %% Path = rpc:call(CTNode, code, get_path, []), + %% [_ | Parts] = lists:reverse(filename:split(DataDir)), + %% TSDir = filename:join(lists:reverse(Parts)), + %% true = rpc:call(CTNode, code, del_path, [TSDir]), + + Mods = [beam_1_SUITE, beam_2_SUITE], + Suites = [atom_to_list(M) || M <- Mods], + [{error,_} = rpc:call(CTNode, code, load_file, [M]) || M <- Mods], + + code:add_path(DataDir), + CRes = + [compile:file(filename:join(DataDir,F), + [verbose,report_errors, + report_warnings,binary]) || F <- Suites], + + [{module,_} = rpc:call(CTNode, code, load_binary, + [Mod, atom_to_list(Mod), Bin]) || + {ok,Mod,Bin} <- CRes], + + {Opts,ERPid} = setup([{suite,Suites},{auto_compile,false}], Config), + + ok = ct_test_support:run(ct, run_test, [Opts], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(beam_me_up, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(beam_me_up, 1), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). +%reformat(Events, _EH) -> +% Events. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(beam_me_up) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{2,2,4}}, + {?eh,tc_start,{beam_1_SUITE,init_per_suite}}, + {?eh,tc_done,{beam_1_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{beam_1_SUITE,tc1}}, + {?eh,tc_done,{beam_1_SUITE,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{beam_1_SUITE,tc2}}, + {?eh,tc_done,{beam_1_SUITE,tc2,{failed,{error,'tc2 failed'}}}}, + {?eh,test_stats,{1,1,{0,0}}}, + {?eh,tc_start,{beam_1_SUITE,end_per_suite}}, + {?eh,tc_done,{beam_1_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{beam_2_SUITE,init_per_suite}}, + {?eh,tc_done,{beam_2_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{beam_2_SUITE,tc1}}, + {?eh,tc_done,{beam_2_SUITE,tc1,ok}}, + {?eh,test_stats,{2,1,{0,0}}}, + {?eh,tc_start,{beam_2_SUITE,tc2}}, + {?eh,tc_done,{beam_2_SUITE,tc2,{failed,{error,'tc2 failed'}}}}, + {?eh,test_stats,{2,2,{0,0}}}, + {?eh,tc_start,{beam_2_SUITE,end_per_suite}}, + {?eh,tc_done,{beam_2_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_misc_1_SUITE_data/beam_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE_data/beam_1_SUITE.erl new file mode 100644 index 0000000000..382bdefded --- /dev/null +++ b/lib/common_test/test/ct_misc_1_SUITE_data/beam_1_SUITE.erl @@ -0,0 +1,134 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_1_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [ + {timetrap,{seconds,10}} + ]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: all() -> TestCases | {skip,Reason} +%% +%% TestCases = [TestCase | {sequence,SeqName}] +%% TestCase = atom() +%% Name of a test case. +%% SeqName = atom() +%% Name of a test case sequence. +%% Reason = term() +%% The reason for skipping all test cases. +%% +%% Description: Returns the list of test cases that are to be executed. +%%-------------------------------------------------------------------- +all() -> + [tc1, tc2]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +tc1(_Config) -> + ct:comment("tc1 executed"), + ok. + +tc2(_Config) -> + exit('tc2 failed'). diff --git a/lib/common_test/test/ct_misc_1_SUITE_data/beam_2_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE_data/beam_2_SUITE.erl new file mode 100644 index 0000000000..70c1f2b471 --- /dev/null +++ b/lib/common_test/test/ct_misc_1_SUITE_data/beam_2_SUITE.erl @@ -0,0 +1,134 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(beam_2_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [ + {timetrap,{seconds,10}} + ]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: all() -> TestCases | {skip,Reason} +%% +%% TestCases = [TestCase | {sequence,SeqName}] +%% TestCase = atom() +%% Name of a test case. +%% SeqName = atom() +%% Name of a test case sequence. +%% Reason = term() +%% The reason for skipping all test cases. +%% +%% Description: Returns the list of test cases that are to be executed. +%%-------------------------------------------------------------------- +all() -> + [tc1, tc2]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +tc1(_Config) -> + ct:comment("tc1 executed"), + ok. + +tc2(_Config) -> + exit('tc2 failed'). diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index 9f428723f5..2e02061dec 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -89,14 +89,14 @@ auto_skip(Config) when is_list(Config) -> ], {Opts,ERPid} = setup({suite,Suites}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(auto_skip, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(auto_skip), + TestEvents = events_to_check(auto_skip), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -112,14 +112,14 @@ user_skip(Config) when is_list(Config) -> Join(DataDir, "user_skip_5_SUITE")], {Opts,ERPid} = setup({suite,Suites}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(user_skip, reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(user_skip), + TestEvents = events_to_check(user_skip), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -142,6 +142,15 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + test_events(auto_skip) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, diff --git a/lib/common_test/test/ct_smoke_test_SUITE.erl b/lib/common_test/test/ct_smoke_test_SUITE.erl index f1c695f614..05a2c20695 100644 --- a/lib/common_test/test/ct_smoke_test_SUITE.erl +++ b/lib/common_test/test/ct_smoke_test_SUITE.erl @@ -162,7 +162,7 @@ dir1(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -170,7 +170,7 @@ dir1(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(dir1), + TestEvents = events_to_check(dir1), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -191,7 +191,7 @@ dir2(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -199,7 +199,7 @@ dir2(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(dir2), + TestEvents = events_to_check(dir2), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -221,7 +221,7 @@ dir1_2(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -229,7 +229,7 @@ dir1_2(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(dir1_2), + TestEvents = events_to_check(dir1_2), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -251,7 +251,7 @@ suite11(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -259,7 +259,7 @@ suite11(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(suite11), + TestEvents = events_to_check(suite11), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -280,7 +280,7 @@ suite21(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -288,7 +288,7 @@ suite21(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(suite21), + TestEvents = events_to_check(suite21), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -311,7 +311,7 @@ suite11_21(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -319,7 +319,7 @@ suite11_21(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(suite11_21), + TestEvents = events_to_check(suite11_21), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -342,7 +342,7 @@ tc111(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -350,7 +350,7 @@ tc111(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(tc111), + TestEvents = events_to_check(tc111), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -372,7 +372,7 @@ tc211(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -380,7 +380,7 @@ tc211(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(tc211), + TestEvents = events_to_check(tc211), ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- @@ -403,7 +403,7 @@ tc111_112(Config) when is_list(Config) -> ERPid = ct_test_support:start_event_receiver(Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -411,7 +411,7 @@ tc111_112(Config) when is_list(Config) -> ct_test_support:reformat(Events, ?eh), ?config(priv_dir, Config)), - TestEvents = test_events(tc111_112), + TestEvents = events_to_check(tc111_112), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -423,8 +423,16 @@ eh_opts(Config) -> Level = ?config(trace_level, Config), [{event_handler,{?eh,[{cbm,ct_test_support},{trace_level,Level}]}}]. +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). -test_events(Test) when Test == dir1 ; Test == dir2 ; +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + events(Test) ++ events_to_check(Test, N-1). + +events(Test) when Test == dir1 ; Test == dir2 ; Test == suite11 ; Test == suite21 -> Suite = if Test == dir1 ; Test == suite11 -> happy_11_SUITE; true -> happy_21_SUITE @@ -465,7 +473,7 @@ test_events(Test) when Test == dir1 ; Test == dir2 ; {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; -test_events(Test) when Test == dir1_2 ; Test == suite11_21 -> +events(Test) when Test == dir1_2 ; Test == suite11_21 -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, @@ -532,7 +540,7 @@ test_events(Test) when Test == dir1_2 ; Test == suite11_21 -> {?eh,stop_logging,[]} ]; -test_events(Test) when Test == tc111 ; Test == tc211 -> +events(Test) when Test == tc111 ; Test == tc211 -> Suite = if Test == tc111 -> happy_11_SUITE; true -> happy_21_SUITE end, [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -549,7 +557,7 @@ test_events(Test) when Test == tc111 ; Test == tc211 -> {?eh,stop_logging,[]} ]; -test_events(tc111_112) -> +events(tc111_112) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 069f8c75fc..eb85409073 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -87,14 +87,14 @@ ts_if_1(Config) when is_list(Config) -> TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir, "ts_if_1_spec"), {Opts,ERPid} = setup({spec,TestSpecName}, Config), - ok = ct_test_support:run(ct, run_test, [Opts], Config), + ok = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(ts_if_1, reformat(Events, ?eh), PrivDir), - TestEvents = test_events(ts_if_1), + TestEvents = events_to_check(ts_if_1), ok = ct_test_support:verify_events(TestEvents, Events, Config). @@ -119,6 +119,15 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + test_events(ts_if_1) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -193,14 +202,14 @@ test_events(ts_if_1) -> {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]}, {?eh,tc_start,{ts_if_1_SUITE,tc12}}, - {?eh,tc_done,{undefined,undefined,{testcase_aborted,{abort_current_testcase,tc12},'_'}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc12,{failed,{testcase_aborted,'stopping tc12'}}}}, {?eh,test_stats,{2,5,{3,6}}}, {?eh,tc_start,{ts_if_1_SUITE,tc13}}, {?eh,tc_done,{ts_if_1_SUITE,tc13,ok}}, {?eh,test_stats,{3,5,{3,6}}}, {?eh,tc_start,{ts_if_1_SUITE,end_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,end_per_suite,ok}}, -%%! + {?eh,tc_start,{ts_if_2_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_2_SUITE,init_per_suite, {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl index 8e90df21ce..47cea190dd 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl @@ -93,6 +93,10 @@ init_per_testcase(_TestCase, Config) -> %%-------------------------------------------------------------------- end_per_testcase(tc2, Config) -> timer:sleep(5000); +end_per_testcase(tc12, Config) -> + ct:comment("end_per_testcase(tc12) called!"), + ct:pal("end_per_testcase(tc12) called!", []), + ok; end_per_testcase(_TestCase, _Config) -> ok. @@ -180,9 +184,9 @@ gtc2(_) -> exit(should_have_been_skipped). tc12(_) -> - F = fun() -> ct:abort_current_testcase({abort_current_testcase,tc12}) end, + F = fun() -> ct:abort_current_testcase('stopping tc12') end, spawn(F), - timer:sleep(500), + timer:sleep(1000), exit(should_have_been_aborted). tc13(_) -> diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 6148e3280e..7bfb9ffb49 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2008-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -23,12 +23,13 @@ %%% -module(ct_test_support). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -include_lib("common_test/include/ct_event.hrl"). -export([init_per_suite/1, init_per_suite/2, end_per_suite/1, - init_per_testcase/2, end_per_testcase/2, write_testspec/3, - run/4, get_opts/1, wait_for_ct_stop/1]). + init_per_testcase/2, end_per_testcase/2, + write_testspec/2, write_testspec/3, + run/2, run/4, get_opts/1, wait_for_ct_stop/1]). -export([handle_event/2, start_event_receiver/1, get_events/2, verify_events/3, reformat/2, log_events/3]). @@ -55,17 +56,28 @@ init_per_suite(Config, Level) -> test_server:fail(Reason); {ok,CTNode} -> test_server:format(0, "Node ~p started~n", [CTNode]), + IsCover = test_server:is_cover(), + if IsCover -> + cover:start(CTNode); + true-> + ok + end, DataDir = ?config(data_dir, Config), PrivDir = ?config(priv_dir, Config), %% PrivDir as well as directory of Test Server suites %% have to be in code path on Common Test node. - true = rpc:call(CTNode, code, add_patha, [PrivDir]), [_ | Parts] = lists:reverse(filename:split(DataDir)), TSDir = filename:join(lists:reverse(Parts)), - true = rpc:call(CTNode, code, add_patha, [TSDir]), - test_server:format(Level, "Dirs added to code path (on ~w):~n" - "~s~n~s~n", [CTNode,TSDir,PrivDir]), + AddPathDirs = case ?config(path_dirs, Config) of + undefined -> []; + Ds -> Ds + end, + PathDirs = [PrivDir,TSDir | AddPathDirs], + [true = rpc:call(CTNode, code, add_patha, [D]) || D <- PathDirs], + test_server:format(Level, "Dirs added to code path (on ~w):~n", + [CTNode]), + [io:format("~s~n", [D]) || D <- PathDirs], TraceFile = filename:join(DataDir, "ct.trace"), case file:read_file_info(TraceFile) of @@ -87,6 +99,7 @@ end_per_suite(Config) -> CTNode = ?config(ct_node, Config), PrivDir = ?config(priv_dir, Config), true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]), + cover:stop(CTNode), slave:stop(CTNode), ok. @@ -95,9 +108,17 @@ end_per_suite(Config) -> init_per_testcase(_TestCase, Config) -> {_,{_,LogDir}} = lists:keysearch(logdir, 1, get_opts(Config)), - test_server:format("See Common Test logs here:\n" - "<a href=\"file://~s/all_runs.html\">~s/all_runs.html</a>", - [LogDir,LogDir]), + case lists:keysearch(master, 1, Config) of + false-> + test_server:format("See Common Test logs here:\n\n" + "<a href=\"file://~s/all_runs.html\">~s/all_runs.html</a>\n" + "<a href=\"file://~s/index.html\">~s/index.html</a>", + [LogDir,LogDir,LogDir,LogDir]); + {value, _}-> + test_server:format("See CT Master Test logs here:\n\n" + "<a href=\"file://~s/master_runs.html\">~s/master_runs.html</a>", + [LogDir,LogDir]) + end, Config. %%%----------------------------------------------------------------- @@ -111,9 +132,10 @@ end_per_testcase(_TestCase, Config) -> %%%----------------------------------------------------------------- %%% - write_testspec(TestSpec, Dir, Name) -> - TSFile = filename:join(Dir, Name), + write_testspec(TestSpec, filename:join(Dir, Name)). + +write_testspec(TestSpec, TSFile) -> {ok,Dev} = file:open(TSFile, [write]), [io:format(Dev, "~p.~n", [Entry]) || Entry <- TestSpec], file:close(Dev), @@ -158,10 +180,32 @@ get_opts(Config) -> %%%----------------------------------------------------------------- %%% +run(Opts, Config) -> + CTNode = ?config(ct_node, Config), + Level = ?config(trace_level, Config), + %% use ct interface + test_server:format(Level, "~n[RUN #1] Calling ct:run_test(~p) on ~p~n", + [Opts, CTNode]), + Result1 = rpc:call(CTNode, ct, run_test, [Opts]), + + %% use run_test interface (simulated) + test_server:format(Level, "Saving start opts on ~p: ~p~n", [CTNode,Opts]), + rpc:call(CTNode, application, set_env, [common_test, run_test_start_opts, Opts]), + test_server:format(Level, "[RUN #2] Calling ct_run:script_start() on ~p~n", [CTNode]), + Result2 = rpc:call(CTNode, ct_run, script_start, []), + case {Result1,Result2} of + {ok,ok} -> + ok; + {E,_} when E =/= ok -> + E; + {_,E} when E =/= ok -> + E + end. + run(M, F, A, Config) -> CTNode = ?config(ct_node, Config), Level = ?config(trace_level, Config), - test_server:format(Level, "Calling ~w:~w(~p) on ~p~n", + test_server:format(Level, "~nCalling ~w:~w(~p) on ~p~n", [M, F, A, CTNode]), rpc:call(CTNode, M, F, A). @@ -231,8 +275,12 @@ verify_events(TEvs, Evs, Config) -> ok end. +verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _) + when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging -> + test_server:format("Failed to find ~p in the list of events!~n", [TestEv]), + exit({event_not_found,TestEv}); + verify_events1(TEvs = [TestEv | TestEvs], Evs = [_|Events], Node, Config) -> -%% test_server:format("Next expected event: ~p~n", [TestEv]), case catch locate(TestEv, Node, Evs, Config) of nomatch -> verify_events1(TEvs, Events, Node, Config); @@ -332,6 +380,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, Func == F -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_start_not_found,TEv}); (_) -> true end, Evs1), @@ -345,6 +397,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, Func == F -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_done_not_found,TEv}); (_) -> true end, Evs2), @@ -388,6 +444,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, EvGName == GroupName, EvProps == Props -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_start_not_found,TEv}); (_) -> true end, RemEvs), @@ -410,6 +470,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, EvGName == GroupName, EvProps == Props, Res == R -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_done_not_found,TEv}); (_) -> true end, RemEvs), @@ -429,6 +493,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> data={Mod,end_per_group,Reason}}}) when EH == TEH, EvNode == Node, Mod == M, Reason == R -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_auto_skip_not_found,TEv}); (_) -> true end, RemEvs), @@ -533,6 +601,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, Func == F -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_start_not_found,TEv}); (_) -> true end, Evs1), @@ -576,6 +648,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, EvGName == GroupName -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_start_not_found,TEv}); (_) -> true end, RemEvs), @@ -611,6 +687,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> EH == TEH, EvNode == Node, Mod == M, EvGName == GroupName, Res == R -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_done_not_found,TEv}); (_) -> true end, RemEvs), @@ -643,6 +723,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> data={Mod,end_per_group,Reason}}}) when EH == TEH, EvNode == Node, Mod == M, Reason == R -> false; + ({EH,#event{name=stop_logging, + node=EvNode,data=_}}) when + EH == TEH, EvNode == Node -> + exit({tc_auto_skip_not_found,TEv}); (_) -> true end, RemEvs), @@ -785,7 +869,8 @@ log_events(TC, Events, PrivDir) -> io:format(Dev, "[~n", []), log_events1(Events, Dev, " "), file:close(Dev), - io:format("Events written to logfile: ~p~n", [LogFile]), + io:format("Events written to logfile: <a href=\"file://~s\">~s</a>~n", + [LogFile,LogFile]), io:format(user, "Events written to logfile: ~p~n", [LogFile]). log_events1(Evs, Dev, "") -> diff --git a/lib/common_test/test/ct_test_support_eh.erl b/lib/common_test/test/ct_test_support_eh.erl index fd3ae18746..70f73b9b81 100644 --- a/lib/common_test/test/ct_test_support_eh.erl +++ b/lib/common_test/test/ct_test_support_eh.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -44,8 +44,20 @@ %% Description: Whenever a new event handler is added to an event manager, %% this function is called to initialize the event handler. %%-------------------------------------------------------------------- +init(String = [X|_]) when is_integer(X) -> + case erl_scan:string(String++".") of + {ok,Ts,_} -> + case erl_parse:parse_term(Ts) of + {ok,Args} -> + init(Args); + _ -> + init(String) + end; + _ -> + init(String) + end; + init(Args) -> - S1 = case lists:keysearch(cbm, 1, Args) of {_,{cbm,CBM}} -> #state{cbm=CBM}; @@ -58,7 +70,8 @@ init(Args) -> _ -> S1 end, - print(S2#state.trace_level, "Event Handler ~w started!~n", [?MODULE]), + print(S2#state.trace_level, "Event Handler ~w started with ~p~n", + [?MODULE,Args]), {ok,S2}. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl new file mode 100644 index 0000000000..dc399bfb4c --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -0,0 +1,433 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_testspec_1_SUITE +%%% +%%% Description: +%%% Test test specifications +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_testspec_1_SUITE). + +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +all(doc) -> + ["Run smoke tests of Common Test."]; + +all(suite) -> + [all_suites, skip_all_suites, + suite, skip_suite, + all_testcases, skip_all_testcases, + testcase, skip_testcase, + all_groups, skip_all_groups, + group, skip_group, + group_all_testcases, skip_group_all_testcases, + group_testcase, skip_group_testcase, + topgroup, + subgroup, skip_subgroup, + subgroup_all_testcases, skip_subgroup_all_testcases, + subgroup_testcase, skip_subgroup_testcase, + sub_skipped_by_top, + testcase_in_multiple_groups]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% + +all_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{label,"all_suites"}, + {suites,TestDir,all}], + + setup_and_execute(all_suites, TestSpec, Config). + +skip_all_suites(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{label,skip_all_suites}, + {suites,TestDir,all}, + {skip_suites,TestDir,all,"SKIPPED!"}], + + setup_and_execute(skip_all_suites, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +suite(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{label,undefined}, + {suites,TestDir,simple_1_SUITE}], + + setup_and_execute(suite, TestSpec, Config). + +skip_suite(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,[simple_1_SUITE,simple_2_SUITE]}, + {skip_suites,TestDir,simple_1_SUITE,"SKIPPED!"}], + + setup_and_execute(skip_suite, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{cases,TestDir,simple_1_SUITE,all}], + + setup_and_execute(all_testcases, TestSpec, Config). + +skip_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,[simple_1_SUITE]}, + {skip_cases,TestDir,simple_1_SUITE,all,"SKIPPED!"}], + + setup_and_execute(skip_all_testcases, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{cases,TestDir,simple_1_SUITE,tc1}], + + setup_and_execute(testcase, TestSpec, Config). + +skip_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{cases,TestDir,simple_1_SUITE,[tc1,tc2]}, + {cases,TestDir,simple_2_SUITE,[tc2,tc1]}, + {skip_cases,TestDir,simple_1_SUITE,[tc1],"SKIPPED!"}, + {skip_cases,TestDir,simple_2_SUITE,tc2,"SKIPPED!"}], + + setup_and_execute(skip_testcase, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,all}], + + setup_and_execute(all_groups, TestSpec, Config). + +skip_all_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,all}, + {skip_groups,TestDir,groups_11_SUITE,all,"SKIPPED!"}], + + setup_and_execute(skip_all_groups, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a}], + + setup_and_execute(group, TestSpec, Config). + +skip_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a, + test_group_1b]}, + {skip_groups,TestDir,groups_11_SUITE, + [test_group_1b,test_group_2,test_group_7],"SKIPPED!"}], + + setup_and_execute(skip_group, TestSpec, Config). + + +%%%----------------------------------------------------------------- +%%% + +group_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,{cases,all}}], + + setup_and_execute(group_all_testcases, TestSpec, Config). + +skip_group_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a, + test_group_1b]}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1b,{cases,all},"SKIPPED!"}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1a,{cases,all},"SKIPPED!"}], + + setup_and_execute(skip_group_all_testcases, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +group_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,{cases,testcase_1a}}], + + setup_and_execute(group_testcase, TestSpec, Config). + +skip_group_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a, + {cases,[testcase_1a,testcase_1b]}}, + {groups,TestDir,groups_11_SUITE,test_group_1b, + {cases,[testcase_1b,testcase_1a]}}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1a,{cases,testcase_1b},"SKIPPED!"}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1b,{cases,[testcase_1a]},"SKIPPED!"}], + + setup_and_execute(skip_group_testcase, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +topgroup(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}, + {groups,TestDir,groups_12_SUITE,test_group_4}], + + setup_and_execute(topgroup, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +subgroup(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_3}], + + setup_and_execute(subgroup, TestSpec, Config). + +skip_subgroup(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_4]}, + {skip_groups,TestDir,groups_12_SUITE, + test_group_8,"SKIPPED!"}], + + setup_and_execute(skip_subgroup, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +subgroup_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE, + test_group_5,{cases,all}}, + {groups,TestDir,groups_12_SUITE, + test_group_3,{cases,all}}], + + setup_and_execute(subgroup_all_testcases, TestSpec, Config). + +skip_subgroup_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_4}, + {skip_groups,TestDir,groups_12_SUITE, + test_group_5,{cases,all},"SKIPPED!"}], + + setup_and_execute(skip_subgroup_all_testcases, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +subgroup_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE, + test_group_7,{cases,testcase_7a}}, + {groups,TestDir,groups_12_SUITE, + test_group_3,{cases,testcase_3b}}], + + setup_and_execute(subgroup_testcase, TestSpec, Config). + +skip_subgroup_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5}, + {skip_groups,TestDir,groups_12_SUITE, + test_group_7,{cases,[testcase_7a,testcase_7b]},"SKIPPED!"}], + + setup_and_execute(skip_subgroup_testcase, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +%%! +%%! Somewhat weird result from this one: +%%! +sub_skipped_by_top(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5}, + {skip_groups,TestDir,groups_12_SUITE,test_group_4,"SKIPPED!"}], + + setup_and_execute(sub_skipped_by_top, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +testcase_in_multiple_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir,groups_12_SUITE,[testcase_1a,testcase_1b]}, + {skip_cases,TestDir,groups_12_SUITE,[testcase_1b],"SKIPPED!"}], + + setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +setup_and_execute(TCName, TestSpec, Config) -> + SpecFile = create_spec_file(?config(priv_dir, Config), + TCName, TestSpec), + TestTerms = + case lists:keymember(label, 1, TestSpec) of + true -> [{spec,SpecFile}]; + false -> [{spec,SpecFile},{label,TCName}] + end, + {Opts,ERPid} = setup(TestTerms, Config), + ok = ct_test_support:run(Opts, Config), + TestSpec1 = [{logdir,proplists:get_value(logdir,Opts)}, + {label,proplists:get_value(label,TestTerms)} | TestSpec], + ok = ct_test_support:run(ct, run_testspec, [TestSpec1], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(TCName, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(TCName), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +create_spec_file(SpecDir, TCName, TestSpec) -> + FileName = filename:join(SpecDir, + atom_to_list(TCName)++".spec"), + {ok,Dev} = file:open(FileName, [write]), + [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec], + file:close(Dev), + io:format("~nTest spec created here~n~n<a href=\"file://~s\">~s</a>~n", + [FileName,FileName]), + FileName. + +setup(Test, Config) when is_tuple(Test) -> + setup([Test], Config); +setup(Tests, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ Tests ++ [{event_handler,{?eh,EvHArgs}}], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). +%reformat(Events, _EH) -> +% Events. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(_) -> + [ + ]. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl new file mode 100644 index 0000000000..4f11d8a0e8 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl @@ -0,0 +1,281 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_11_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [testcase_1a,testcase_1b]}, + + {test_group_1b, [], [testcase_1a,testcase_1b]}, + + {test_group_2, [], [testcase_2a, + + {test_group_3, [], [testcase_3a, + testcase_3b]}, + testcase_2b]}, + + {test_group_4, [{test_group_5, [], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + {test_group_6, [{group, test_group_7}]}, + + {test_group_7, [testcase_7a,testcase_7b]} + ]. + +all() -> + [testcase_1, + {group, test_group_1a}, + {group, test_group_1b}, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1, + testcase_1a, testcase_1b, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + [{name,Group}] = ?config(tc_group_properties,Config), + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(Group), + init = ?config(suite,Config), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(Group), + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl new file mode 100644 index 0000000000..69c06f9b83 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl @@ -0,0 +1,344 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_12_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]}, + + {test_group_1b, [parallel], [testcase_1a,testcase_1b]}, + + {test_group_2, [parallel], [testcase_2a, + + {test_group_3, [{repeat,2}], + [testcase_3a, testcase_3b]}, + + testcase_2b]}, + + {test_group_4, [{test_group_5, [parallel], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [parallel], [{group, test_group_7}, + {group, test_group_8}]}, + + {test_group_7, [sequence], [testcase_7a,testcase_7b]}, + + {test_group_8, [shuffle,sequence], [testcase_8a,testcase_8b]} + ]. + +all() -> + [{group, test_group_1a}, + {group, test_group_1b}, + testcase_1, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7, + test_group_8 + ], + [ + testcase_1a, testcase_1b, testcase_1c, + testcase_1, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b, + testcase_8a, testcase_8b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + Cmt = + case {Group,?config(tc_group_properties,Config)} of + {test_group_1a,[{shuffle,S},{name,test_group_1a}]} -> + io_lib:format("shuffled, ~w", [S]); + {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; + {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; + {test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2"; + {test_group_3,[{name,test_group_3}]} -> "repeat 1"; + {test_group_4,[{name,test_group_4}]} -> ok; + {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; + {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; + {test_group_7,[{name,test_group_7},sequence]} -> "sequence"; + {test_group_8,[{shuffle,_},{name,test_group_8},sequence]} -> + "shuffle & sequence" + end, + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + ct:comment(io_lib:format("~w, ~s", [Group,Cmt])), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_1c() -> + []. +testcase_1c(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1b,Config), + testcase_1c = ?config(testcase_1c,Config), + ok. + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + %% increase chance the done event will come + %% during execution of subgroup (could be + %% tricky to handle) + timer:sleep(3), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. + +testcase_8a() -> + []. +testcase_8a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_8 = ?config(test_group_8,Config), + testcase_8a = ?config(testcase_8a,Config), + ok. +testcase_8b() -> + []. +testcase_8b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_8 = ?config(test_group_8,Config), + undefined = ?config(testcase_8a,Config), + testcase_8b = ?config(testcase_8b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl new file mode 100644 index 0000000000..2533ac8e84 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl @@ -0,0 +1,281 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_21_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [testcase_1a,testcase_1b]}, + + {test_group_1b, [], [testcase_1a,testcase_1b]}, + + {test_group_2, [], [testcase_2a, + + {test_group_3, [], [testcase_3a, + testcase_3b]}, + testcase_2b]}, + + {test_group_4, [{test_group_5, [], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [{group, test_group_7}]}, + + {test_group_7, [testcase_7a,testcase_7b]} + ]. + +all() -> + [testcase_1, + {group, test_group_1a}, + {group, test_group_1b}, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1, + testcase_1a, testcase_1b, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + [{name,Group}] = ?config(tc_group_properties,Config), + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(io_lib:format("~w", [Group])), + init = ?config(suite,Config), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl new file mode 100644 index 0000000000..cd517876df --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl @@ -0,0 +1,314 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_22_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]}, + + {test_group_1b, [parallel], [testcase_1a,testcase_1b]}, + + {test_group_2, [parallel], [testcase_2a, + + {test_group_3, [{repeat,1}], + [testcase_3a, testcase_3b]}, + + testcase_2b]}, + + {test_group_4, [{test_group_5, [parallel], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [parallel], [{group, test_group_7}]}, + + {test_group_7, [sequence], [testcase_7a,testcase_7b]} + ]. + +all() -> + [{group, test_group_1a}, + {group, test_group_1b}, + testcase_1, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1a, testcase_1b, testcase_1c, + testcase_1, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + Cmt = + case {Group,?config(tc_group_properties,Config)} of + {test_group_1a,[{shuffle,S},{name,test_group_1a}]} -> + io_lib:format("shuffled, ~w", [S]); + {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; + {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; + {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1"; + {test_group_3,[{name,test_group_3}]} -> "repeat 0"; + {test_group_4,[{name,test_group_4}]} -> ok; + {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; + {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; + {test_group_7,[{name,test_group_7},sequence]} -> "sequence" + end, + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + ct:comment(io_lib:format("~w, ~s", [Group,Cmt])), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_1c() -> + []. +testcase_1c(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1b,Config), + testcase_1c = ?config(testcase_1c,Config), + ok. + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + %% increase chance the done event will come + %% during execution of subgroup (could be + %% tricky to handle) + timer:sleep(3), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl new file mode 100644 index 0000000000..b789851134 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl @@ -0,0 +1,146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(simple_1_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [ + {timetrap,{seconds,10}} + ]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + [{ips,ips_data} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ips_data = ?config(ips, Config). + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(TestCase, Config) -> + [{TestCase,{TestCase,data}} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(TestCase, Config) -> + {TestCase,data} = ?config(TestCase, Config). + +%%-------------------------------------------------------------------- +%% Function: all() -> TestCases | {skip,Reason} +%% +%% TestCases = [TestCase | {sequence,SeqName}] +%% TestCase = atom() +%% Name of a test case. +%% SeqName = atom() +%% Name of a test case sequence. +%% Reason = term() +%% The reason for skipping all test cases. +%% +%% Description: Returns the list of test cases that are to be executed. +%%-------------------------------------------------------------------- +all() -> + [tc1, + tc2]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +tc1() -> + [{userdata,{info, "This is a testcase"}}]. + +tc1(Config) -> + ips_data = ?config(ips, Config), + {tc1,data} = ?config(tc1, Config), + ok. + +tc2() -> + [{timetrap,5000}]. + +tc2(Config) -> + ips_data = ?config(ips, Config), + undefined = ?config(tc1, Config), + {tc2,data} = ?config(tc2, Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl new file mode 100644 index 0000000000..eb7e9cdf7b --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl @@ -0,0 +1,146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(simple_2_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [ + {timetrap,{seconds,10}} + ]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + [{ips,ips_data} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ips_data = ?config(ips, Config). + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(TestCase, Config) -> + [{TestCase,{TestCase,data}} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(TestCase, Config) -> + {TestCase,data} = ?config(TestCase, Config). + +%%-------------------------------------------------------------------- +%% Function: all() -> TestCases | {skip,Reason} +%% +%% TestCases = [TestCase | {sequence,SeqName}] +%% TestCase = atom() +%% Name of a test case. +%% SeqName = atom() +%% Name of a test case sequence. +%% Reason = term() +%% The reason for skipping all test cases. +%% +%% Description: Returns the list of test cases that are to be executed. +%%-------------------------------------------------------------------- +all() -> + [tc1, + tc2]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +tc1() -> + [{userdata,{info, "This is a testcase"}}]. + +tc1(Config) -> + ips_data = ?config(ips, Config), + {tc1,data} = ?config(tc1, Config), + ok. + +tc2() -> + [{timetrap,5000}]. + +tc2(Config) -> + ips_data = ?config(ips, Config), + undefined = ?config(tc1, Config), + {tc2,data} = ?config(tc2, Config), + ok. diff --git a/lib/common_test/test/ct_userconfig_callback.erl b/lib/common_test/test/ct_userconfig_callback.erl new file mode 100644 index 0000000000..ca51bf240b --- /dev/null +++ b/lib/common_test/test/ct_userconfig_callback.erl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +-module(ct_userconfig_callback). + +-export([check_parameter/1, read_config/1]). + +read_config(Str) -> + KeyVals = string:tokens(Str, " "), + {ok,read_config1(KeyVals)}. + +read_config1([Key,Val | KeyVals]) -> + [{list_to_atom(Key),Val} | read_config1(KeyVals)]; +read_config1([]) -> + []. + +check_parameter(Str) -> + {ok,{config,Str}}. diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index ee07350c55..413ef21df3 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1,3 +1,3 @@ -COMMON_TEST_VSN = 1.4.7 +COMMON_TEST_VSN = 1.5.1 |
