diff options
| author | Lukas Larsson <[email protected]> | 2011-10-04 10:16:09 +0200 |
|---|---|---|
| committer | Lukas Larsson <[email protected]> | 2011-10-04 10:16:09 +0200 |
| commit | 2d25f96f904bfa33c4bba4f3026f9fd2117ed8ab (patch) | |
| tree | 26acbda51fa2ac4dd5f809e400d05aa1021eba20 /lib | |
| parent | 622daf3b96666f8bbabec44a15b26a188a83b95e (diff) | |
| parent | 1a42cde6d62a8ec0f9e22215d3c3ec655201c78f (diff) | |
| download | otp-2d25f96f904bfa33c4bba4f3026f9fd2117ed8ab.tar.gz otp-2d25f96f904bfa33c4bba4f3026f9fd2117ed8ab.tar.bz2 otp-2d25f96f904bfa33c4bba4f3026f9fd2117ed8ab.zip | |
Merge branch 'dev' into major
* dev: (38 commits)
Update documentation
Rid ct_telnet of doc build warnings
Create temporary fix for problem with parallel test cases
Update primary bootstrap
Correct "Missing Suites" link
Add documentation on timetraps and start flags
Add missing tests for timetrap handling and fix remaining errors
Solve problem with ct_init/end_per_group being counted as test cases
Fix errors in test suites
Fix invalid call to undefined function
Fix problem with test_server_ctrl creating invalid conf test
Improve info in CT framework log
Update vsn.mk for common_test and test_server
Enhance logging performance
Change order of include files
Add link to last executed test suite on index page
Fix problem with location value when init config func calls help func
Fix crash when CTHook init fails
Correct error in test suite
Fix error with incorrect notification after end_per_testcase craches
...
Conflicts:
bootstrap/bin/start.boot
bootstrap/bin/start_clean.boot
bootstrap/lib/compiler/ebin/beam_asm.beam
bootstrap/lib/compiler/ebin/beam_disasm.beam
bootstrap/lib/compiler/ebin/compile.beam
bootstrap/lib/compiler/ebin/sys_pre_expand.beam
bootstrap/lib/kernel/ebin/code.beam
bootstrap/lib/kernel/ebin/code_server.beam
bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
bootstrap/lib/kernel/ebin/inet.beam
bootstrap/lib/kernel/ebin/inet_config.beam
bootstrap/lib/kernel/ebin/inet_dns.beam
bootstrap/lib/stdlib/ebin/beam_lib.beam
bootstrap/lib/stdlib/ebin/dets.beam
bootstrap/lib/stdlib/ebin/erl_compile.beam
bootstrap/lib/stdlib/ebin/erl_internal.beam
bootstrap/lib/stdlib/ebin/erl_scan.beam
bootstrap/lib/stdlib/ebin/erl_tar.beam
bootstrap/lib/stdlib/ebin/io_lib_fread.beam
bootstrap/lib/stdlib/ebin/otp_internal.beam
bootstrap/lib/stdlib/ebin/sofs.beam
bootstrap/lib/stdlib/ebin/supervisor.beam
bootstrap/lib/stdlib/ebin/zip.beam
lib/common_test/src/ct.erl
lib/common_test/src/ct_run.erl
lib/common_test/test/ct_error_SUITE.erl
lib/common_test/test/ct_repeat_1_SUITE.erl
lib/common_test/test/ct_skip_SUITE.erl
lib/test_server/src/test_server.erl
Diffstat (limited to 'lib')
36 files changed, 2096 insertions, 521 deletions
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 57b032b3fd..f58b2ab0a9 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -133,9 +133,15 @@ {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns} | {stylesheet,CSSFile} | {ct_hooks, CTHs}</v> - <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} - | {hours,integer()}</v> + <v> Time = TimeVal | TimeFunc</v> + <v> TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | + {hours,integer()}</v> + <v> TimeFunc = {Mod,Func,Args} | Fun</v> <v> MilliSec = integer()</v> + <v> Mod = atom()</v> + <v> Func = atom()</v> + <v> Args = list()</v> + <v> Fun = fun()</v> <v> Required = Key | {Key,SubKeys}</v> <v> Key = atom()</v> <v> SubKeys = SubKey | [SubKey]</v> @@ -161,7 +167,9 @@ test case is allowed to take (including <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c>). If the timetrap time is exceeded, the test case fails with reason - <c>timetrap_timeout</c>.</p> + <c>timetrap_timeout</c>. If a <c>TimeFunc</c> function is specified, + it will be called initially and must return a value on + <c>TimeVal</c> format.</p> <p>The <c>require</c> tag specifies configuration variables that are required by test cases in the suite. If the required @@ -248,7 +256,7 @@ </type> <desc> - <p> MANDATORY (only if one or more groups are defined) </p> + <p> OPTIONAL </p> <p>This function is called before execution of a test case group. It typically contains initialization which is common for @@ -279,7 +287,7 @@ </type> <desc> - <p> MANDATORY (only if one or more groups are defined) </p> + <p> OPTIONAL </p> <p>This function is called after the execution of a test case group is finished. It is meant to be used for cleaning up after <c>init_per_group/2</c>. @@ -353,9 +361,15 @@ <v> Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns}</v> - <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} - | {hours,integer()}</v> + <v> Time = TimeVal | TimeFunc</v> + <v> TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | + {hours,integer()}</v> + <v> TimeFunc = {Mod,Func,Args} | Fun</v> <v> MilliSec = integer()</v> + <v> Mod = atom()</v> + <v> Func = atom()</v> + <v> Args = list()</v> + <v> Fun = fun()</v> <v> Required = Key | {Key,SubKeys}</v> <v> Key = atom()</v> <v> SubKeys = SubKey | [SubKey]</v> @@ -378,7 +392,9 @@ exceeded, the test case fails with reason <c>timetrap_timeout</c>. <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c> are included in the - timetrap time.</p> + timetrap time. If a <c>TimeFunc</c> function is specified, + it will be called before the test case (or <c>init_per_testcase/2</c>) + and must return a value on <c>TimeVal</c> format.</p> <p>The <c>require</c> tag specifies configuration variables that are required by the test case. If the required diff --git a/lib/common_test/doc/src/config_file_chapter.xml b/lib/common_test/doc/src/config_file_chapter.xml index 59151a73ec..6fc6638bf7 100644 --- a/lib/common_test/doc/src/config_file_chapter.xml +++ b/lib/common_test/doc/src/config_file_chapter.xml @@ -285,7 +285,7 @@ <c>{ok, Config}</c> - if the configuration variables are read successfully, </item> <item> - <c>{error, Error, ErrorDetails}</c> - if the callback module fails to + <c>{error, {Error, ErrorDetails}}</c> - if the callback module fails to proceed with the given configuration parameters. </item> </list> @@ -422,14 +422,14 @@ stop()-> call(Client, Request)-> case whereis(?REGISTERED_NAME) of undefined-> - {error, not_started, Request}; + {error, {not_started, Request}}; Pid-> Pid ! {Client, Request}, receive Reply-> {ok, Reply} after 4000-> - {error, timeout, Request} + {error, {timeout, Request}} end end. diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 1ab563d74f..9045646733 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -83,7 +83,7 @@ <title>Run tests from command line</title> <pre> ct_run [-dir TestDir1 TestDir2 .. TestDirN] | - [-suite Suite1 Suite2 .. SuiteN + [[-dir TestDir] -suite Suite1 Suite2 .. SuiteN [[-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]]] [-step [config | keep_inactive]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] @@ -92,6 +92,7 @@ [-decrypt_key Key] | [-decrypt_file KeyFile] [-label Label] [-logdir LogDir] + [-logopts LogOpts] [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] @@ -117,6 +118,7 @@ [-decrypt_key Key] | [-decrypt_file KeyFile] [-label Label] [-logdir LogDir] + [-logopts LogOpts] [-allow_user_terms] [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] @@ -138,10 +140,11 @@ <pre> ct_run -vts [-browser Browser] [-dir TestDir1 TestDir2 .. TestDirN] | - [-suite Suite [[-group Group] [-case Case]]] + [[dir TestDir] -suite Suite [[-group Group] [-case Case]]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 ConfigString2 and .. and CallbackModuleN ConfigStringN] + [-logopts LogOpts] [-decrypt_key Key] | [-decrypt_file KeyFile] [-include InclDir1 InclDir2 .. InclDirN] [-no_auto_compile] diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 816aa5b1eb..57059f0ba2 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -128,6 +128,15 @@ <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE -case start stop</c></p> <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE -group installation -case start stop</c></p> + <p>It is also possible to combine the <c>dir</c>, <c>suite</c> and <c>group/case</c> flags. E.g, to run + <c>x_SUITE</c> and <c>y_SUITE</c> in directory <c>testdir</c>:</p> + + <p><c>$ ct_run -dir ./testdir -suite x_SUITE y_SUITE</c></p> + + <p>This has the same effect as calling:</p> + + <p><c>$ ct_run -suite ./testdir/x_SUITE ./testdir/y_SUITE</c></p> + <p>Other flags that may be used with <c>ct_run</c>:</p> <list> <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item> @@ -167,6 +176,8 @@ <item><c><![CDATA[-decrypt_file <key_file>]]></c>, points out a file containing a decryption key for <seealso marker="config_file_chapter#encrypted_config_files">encrypted configuration files</seealso>.</item> <item><c><![CDATA[-basic_html]]></c>, switches off html enhancements that might not be compatible with older browsers.</item> + <item><c><![CDATA[-logopts <opts>]]></c>, makes it possible to modify aspects of the logging behaviour, see + <seealso marker="run_test_chapter#logopts">Log options</seealso> below.</item> </list> <note><p>Directories passed to Common Test may have either relative or absolute paths.</p></note> @@ -324,8 +335,9 @@ are to be executed by Common Test, and those functions only. If the step option <c>config</c> is specified, breakpoints will also be initially set on the configuration functions in the suite, i.e. - <c>init_per_suite/1</c>, <c>end_per_suite/1</c>, <c>init_per_testcase/2</c> - and <c>end_per_testcase/2</c>.</p> + <c>init_per_suite/1</c>, <c>end_per_suite/1</c>, + <c>init_per_group/2</c>, <c>end_per_group/2</c>, + <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c>.</p> <p>Common Test enables the Debugger auto attach feature, which means that for every new interpreted test case function that starts to execute, a new trace window will automatically pop up. (This is because each test @@ -652,6 +664,30 @@ to follow test progress simply by refreshing pages in the HTML browser. Statistics totals are not presented until a test is complete however.</p> + <section> + <marker id="logopts"></marker> + <title>Log options</title> + <p>With the <c>logopts</c> start flag, it's possible to specify + options that modify some aspects of the logging behaviour. + Currently, the following options are available:</p> + <list> + <item><c>no_src</c></item> + <item><c>no_nl</c></item> + </list> + <p>With <c>no_src</c>, the html version of the test suite source + code will not be generated during the test run (and consequently + not be available in the log file system).</p> + <p>With <c>no_nl</c>, Common Test will not add a newline character + (\n) to the end of an output string that it receives from a call to e.g. + <c>io:format/2</c>, and which it prints to the test case log.</p> + <p>For example, if a test is started with:</p> + <p><c>$ ct_run -suite my_SUITE -logopts no_src</c></p> + <p>then printouts during the test made by successive calls to <c>io:format("x")</c>, + will appear in the test case log as:</p> + <p><c>xxx</c></p> + <p>instead of each <c>x</c> printed on a new line, which is the default behaviour.</p> + </section> + </section> <section> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 3f9fdb7121..e35888e68f 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -280,6 +280,8 @@ the timetrap time is exceeded, the test case fails with reason <c>timetrap_timeout</c>. Note that <c>init_per_testcase</c> and <c>end_per_testcase</c> are included in the timetrap time. + Please see the <seealso marker="write_test_chapter#timetraps">Timetrap</seealso> + section for more details. </p> </item> <tag><em><c>userdata</c></em></tag> @@ -699,8 +701,8 @@ </section> <section> - <title>Timetrap timeouts</title> <marker id="timetraps"></marker> + <title>Timetrap timeouts</title> <p>The default time limit for a test case is 30 minutes, unless a <c>timetrap</c> is specified either by the suite info function or a test case info function. The timetrap timeout value defined @@ -723,6 +725,13 @@ multipled by <c>multiply_timetraps</c>, and possibly scaled up if <c>scale_timetraps</c> is enabled, the function <c>ct:sleep/1</c> may be called.</p> + <p>A function (<c>fun</c> or <c>MFA</c>) may be specified as timetrap value + in the suite- and test case info function, e.g:</p> + <p><c>{timetrap,{test_utils,get_timetrap_value,[?MODULE,system_start]}}</c></p> + <p>The function will be called initially by Common Test (before execution + of the suite or the test case) and must return a time value such as an + integer (millisec), or a <c>{SecMinOrHourTag,Time}</c> tuple. More + information can be found in the <c>common_test</c> reference manual.</p> </section> <section> @@ -818,6 +827,3 @@ </list> </section> </chapter> - - - diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 3a96190256..69e15fa246 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -148,10 +148,10 @@ run(TestDirs) -> %%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} | %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | -%%% {refresh_logs,LogDir} | {basic_html,Bool} | +%%% {refresh_logs,LogDir} | {logopts,LogOpts} | {basic_html,Bool} | %%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} %%% TestDirs = [string()] | string() -%%% Suites = [string()] | string() +%%% Suites = [string()] | [atom()] | string() | atom() %%% Cases = [atom()] | atom() %%% Groups = [atom()] | atom() %%% TestSpecs = [string()] | string() @@ -177,6 +177,8 @@ run(TestDirs) -> %%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile} %%% DecryptKey = string() %%% DecryptFile = string() +%%% LogOpts = [LogOpt] +%%% LogOpt = no_nl | no_src %%% CTHs = [CTHModule | {CTHModule, CTHInitArgs}] %%% CTHModule = atom() %%% CTHInitArgs = term() diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index 6b75937668..fc51aea7f3 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -204,9 +204,9 @@ get_config_file_list(Opts) -> DefaultConfigs = process_default_configs(Opts), CfgFiles = if - DefaultConfigs == []-> + DefaultConfigs == [] -> []; - true-> + true -> [{?ct_config_txt, DefaultConfigs}] end ++ process_user_configs(Opts, []), @@ -240,12 +240,12 @@ read_config_files(Opts) -> end, ConfigFiles = case lists:keyfind(config, 1, Opts) of - {config,ConfigLists}-> + {config,ConfigLists} -> lists:foldr(fun({Callback,Files}, Acc) -> AddCallback(Callback,Files) ++ Acc end,[],ConfigLists); - false-> + false -> [] end, read_config_files_int(ConfigFiles, fun store_config/3). @@ -255,7 +255,9 @@ read_config_files_int([{Callback, File}|Files], FunToSave) -> {ok, Config} -> FunToSave(Config, Callback, File), read_config_files_int(Files, FunToSave); - {error, ErrorName, ErrorDetail}-> + {error, {ErrorName, ErrorDetail}} -> + {user_error, {ErrorName, File, ErrorDetail}}; + {error, ErrorName, ErrorDetail} -> {user_error, {ErrorName, File, ErrorDetail}} end; read_config_files_int([], _FunToSave) -> @@ -283,7 +285,7 @@ rewrite_config(Config, Callback, File) -> config=File,_='_'}), Updater = fun({Key, Value}) -> case keyfindall(Key, #ct_conf.key, OldRows) of - []-> + [] -> ets:insert(?attr_table, #ct_conf{key=Key, value=Value, @@ -453,9 +455,9 @@ update_conf(Name, NewConfig) -> reload_conf(KeyOrName) -> case lookup_handler_for_config(KeyOrName) of - []-> + [] -> undefined; - HandlerList-> + HandlerList -> HandlerList2 = lists:usort(HandlerList), read_config_files_int(HandlerList2, fun rewrite_config/3), get_config(KeyOrName) @@ -711,13 +713,13 @@ random_bytes_1(N, Acc) -> random_bytes_1(N-1, [random:uniform(255)|Acc]). check_callback_load(Callback) -> case code:is_loaded(Callback) of - {file, _Filename}-> + {file, _Filename} -> check_exports(Callback); - false-> + false -> case code:load_file(Callback) of - {module, Callback}-> + {module, Callback} -> check_exports(Callback); - {error, Error}-> + {error, Error} -> {error, Error} end end. @@ -745,14 +747,14 @@ check_config_files(Configs) -> end, Files) end; - {error, Why}-> + {error, Why} -> {error, {callback, {Callback,Why}}} end; ({Callback, []}) -> case check_callback_load(Callback) of - {ok, Callback}-> + {ok, Callback} -> Callback:check_parameter([]); - {error, Why}-> + {error, Why} -> {error, {callback, {Callback,Why}}} end end, @@ -773,15 +775,15 @@ prepare_user_configs([], Acc, _) -> prepare_config_list(Args) -> ConfigFiles = case lists:keysearch(ct_config, 1, Args) of - {value,{ct_config,Files}}-> + {value,{ct_config,Files}} -> [{?ct_config_txt,[filename:absname(F) || F <- Files]}]; - false-> + false -> [] end, UserConfigs = case lists:keysearch(userconfig, 1, Args) of - {value,{userconfig,UserConfigFiles}}-> + {value,{userconfig,UserConfigFiles}} -> prepare_user_configs(UserConfigFiles, [], new); - false-> + false -> [] end, ConfigFiles ++ UserConfigs. diff --git a/lib/common_test/src/ct_config_plain.erl b/lib/common_test/src/ct_config_plain.erl index 3fbc8af9fb..6698332379 100644 --- a/lib/common_test/src/ct_config_plain.erl +++ b/lib/common_test/src/ct_config_plain.erl @@ -29,7 +29,7 @@ read_config(ConfigFile) -> {ok,Config} -> {ok, Config}; {error,enoent} -> - {error, config_file_error, enoent}; + {error,{config_file_error,file:format_error(enoent)}}; {error,Reason} -> Key = case application:get_env(common_test, decrypt) of @@ -45,23 +45,27 @@ read_config(ConfigFile) -> end, case Key of {error,no_crypt_file} -> - {error, config_file_error, Reason}; + {error,{config_file_error, + lists:flatten( + io_lib:format("~s",[file:format_error(Reason)]))}}; {error,CryptError} -> - {error, decrypt_file_error, CryptError}; + {error,{decrypt_file_error,CryptError}}; _ when is_list(Key) -> - case ct_config:decrypt_config_file(ConfigFile, undefined, {key,Key}) of + case ct_config:decrypt_config_file(ConfigFile, + undefined, + {key,Key}) of {ok,CfgBin} -> case read_config_terms(CfgBin) of {error,ReadFail} -> - {error, config_file_error, ReadFail}; + {error,{config_file_error,ReadFail}}; Config -> - {ok, Config} + {ok,Config} end; {error,DecryptFail} -> - {error, decrypt_config_error, DecryptFail} + {error,{decrypt_config_error,DecryptFail}} end; _ -> - {error, bad_decrypt_key, Key} + {error,{bad_decrypt_key,Key}} end end. @@ -69,9 +73,9 @@ read_config(ConfigFile) -> check_parameter(File)-> case filelib:is_file(File) of true-> - {ok, {file, File}}; + {ok,{file,File}}; false-> - {error, {nofile, File}} + {error,{nofile,File}} end. read_config_terms(Bin) when is_binary(Bin) -> diff --git a/lib/common_test/src/ct_config_xml.erl b/lib/common_test/src/ct_config_xml.erl index 8a6e75e635..794174e663 100644 --- a/lib/common_test/src/ct_config_xml.erl +++ b/lib/common_test/src/ct_config_xml.erl @@ -27,30 +27,30 @@ % read config file read_config(ConfigFile) -> case catch do_read_xml_config(ConfigFile) of - {ok, Config}-> - {ok, Config}; - {error, Error, ErroneousString}-> - {error, Error, ErroneousString} + {ok,Config} -> + {ok,Config}; + Error = {error,_} -> + Error end. % check file exists -check_parameter(File)-> +check_parameter(File) -> case filelib:is_file(File) of - true-> - {ok, {file, File}}; - false-> - {error, {nofile, File}} + true -> + {ok,{file,File}}; + false -> + {error,{nofile,File}} end. % actual reading of the config -do_read_xml_config(ConfigFile)-> +do_read_xml_config(ConfigFile) -> case catch xmerl_sax_parser:file(ConfigFile, - [{event_fun, fun event/3}, - {event_state, []}]) of - {ok, EntityList, _}-> - {ok, lists:reverse(transform_entity_list(EntityList))}; - Oops-> - {error, parsing_failed, Oops} + [{event_fun,fun event/3}, + {event_state,[]}]) of + {ok,EntityList,_} -> + {ok,lists:reverse(transform_entity_list(EntityList))}; + Oops -> + {error,{parsing_failed,Oops}} end. % event callback for xmerl_sax_parser @@ -92,18 +92,18 @@ tag(_El, State) -> State. % transform of the ugly deeply nested entity list to the key-value "tree" -transform_entity_list(EntityList)-> +transform_entity_list(EntityList) -> lists:map(fun transform_entity/1, EntityList). % transform entity from {list(), list()} to {atom(), term()} transform_entity({Tag, [Value|Rest]}) when - is_tuple(Value)-> + is_tuple(Value) -> {list_to_atom(Tag), transform_entity_list(lists:reverse([Value|Rest]))}; -transform_entity({Tag, String})-> +transform_entity({Tag, String}) -> case list_to_term(String) of - {ok, Value}-> + {ok, Value} -> {list_to_atom(Tag), Value}; - Error-> + Error -> throw(Error) end. @@ -111,8 +111,8 @@ transform_entity({Tag, String})-> list_to_term(String) -> {ok, T, _} = erl_scan:string(String++"."), case catch erl_parse:parse_term(T) of - {ok, Term} -> - {ok, Term}; + {ok,Term} -> + {ok,Term}; Error -> - {error, Error, String} + {error,{Error,String}} end. diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 2ebc6c311a..482c5242ce 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -24,10 +24,10 @@ -module(ct_framework). --export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, report/2, warn/1]). --export([error_notification/4]). +-export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]). +-export([report/2, warn/1, error_notification/4]). --export([overview_html_header/1]). +-export([get_logopts/0, format_comment/1, overview_html_header/1]). -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). @@ -116,7 +116,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> Config = lists:keydelete(watchdog,1,Config1), if Func /= init_per_suite, DoInit /= true -> ok; - true -> + true -> %% delete all default values used in previous suite ct_config:delete_default_config(suite), %% release all name -> key bindings (once per suite) @@ -133,7 +133,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> ct_config:delete_default_config(testcase), case add_defaults(Mod,Func,TestCaseInfo,DoInit) of Error = {suite0_failed,_} -> - ct_logs:init_tc(), + ct_logs:init_tc(false), FuncSpec = group_or_func(Func,Config0), ct_event:notify(#event{name=tc_start, node=node(), @@ -143,7 +143,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} when DoInit == false -> - ct_logs:init_tc(), + ct_logs:init_tc(false), FuncSpec = group_or_func(Func,Config0), ct_event:notify(#event{name=tc_start, node=node(), @@ -194,19 +194,24 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> Conns -> ct_util:silence_connections(Conns) end, - - ct_logs:init_tc(), + if Func /= init_per_suite, DoInit /= true -> + ct_logs:init_tc(false); + true -> + ct_logs:init_tc(true) + end, FuncSpec = group_or_func(Func,Config), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), - case configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of + case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of {suite0_failed,Reason} -> ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}), {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; + {'EXIT',Reason} -> + {auto_skip,Reason}; {ok, FinalConfig} -> case MergeResult of {error,Reason} -> @@ -245,7 +250,12 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> Error = {error,_} -> {SuiteInfo,Error}; MergedInfo -> {SuiteInfo,MergedInfo} end; - {'EXIT',Reason} -> + {'EXIT',Reason} -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:suite/0 failed: ~p~n", + [Mod,Reason]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,{exited,Reason}}; SuiteInfo when is_list(SuiteInfo) -> case lists:all(fun(E) when is_tuple(E) -> true; @@ -261,9 +271,19 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> MergedInfo -> {SuiteInfo1,MergedInfo} end; false -> + ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:suite/0: ~p~n", [Mod,SuiteInfo]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end; - _ -> + SuiteInfo -> + ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:suite/0: ~p~n", [Mod,SuiteInfo]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end. @@ -451,7 +471,6 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> {value,{watchdog,Dog}} -> test_server:timetrap_cancel(Dog); false -> ok end, - %% save the testcase process pid so that it can be used %% to look up the attached trace window later case ct_util:get_testdata(interpret) of @@ -461,7 +480,6 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> _ -> ok end, - ct_util:delete_testdata(comment), ct_util:delete_suite_data(last_saved_config), FuncSpec = @@ -767,6 +785,37 @@ get_suite(Mod, Name) -> %%%----------------------------------------------------------------- +get_all_cases(Suite) -> + case get_suite(Suite, all) of + [{?MODULE,error_in_suite,[[{error,_}=Error]]}] -> + Error; + [{?MODULE,error_in_suite,[[Error]]}] -> + {error,Error}; + Tests -> + Cases = get_all_cases1(Suite, Tests), + lists:reverse( + lists:foldl(fun(TC, TCs) -> + case lists:member(TC, TCs) of + true -> TCs; + false -> [TC | TCs] + end + end, [], Cases)) + end. + +get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) -> + get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests); + +get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) -> + [{Suite,Test} | get_all_cases1(Suite, Tests)]; + +get_all_cases1(Suite, [Test | Tests]) -> + [Test | get_all_cases1(Suite, Tests)]; + +get_all_cases1(_, []) -> + []. + +%%%----------------------------------------------------------------- + find_groups(Mod, Name, TCs, GroupDefs) -> Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), trim(Found). @@ -978,15 +1027,20 @@ make_conf(Mod, Name, Props, TestSpec) -> _ -> ok end, - {InitConf,EndConf} = + {InitConf,EndConf,ExtraProps} = case erlang:function_exported(Mod,init_per_group,2) of true -> - {{Mod,init_per_group},{Mod,end_per_group}}; + {{Mod,init_per_group},{Mod,end_per_group},[]}; false -> + ct_logs:log("TEST INFO", "init_per_group/2 and " + "end_per_group/2 missing for group " + "~p in ~p, using default.", + [Name,Mod]), {{?MODULE,ct_init_per_group}, - {?MODULE,ct_end_per_group}} + {?MODULE,ct_end_per_group}, + [{suite,Mod}]} end, - {conf,[{name,Name}|Props],InitConf,TestSpec,EndConf}. + {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. %%%----------------------------------------------------------------- @@ -1159,13 +1213,15 @@ error_in_suite(Config) -> %% if the group config functions are missing in the suite, %% use these instead ct_init_per_group(GroupName, Config) -> - ct_logs:log("WARNING", "init_per_group/2 for ~w missing " + ct:comment(io_lib:format("start of ~p", [GroupName])), + ct_logs:log("TEST INFO", "init_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), Config. ct_end_per_group(GroupName, _) -> - ct_logs:log("WARNING", "end_per_group/2 for ~w missing " + ct:comment(io_lib:format("end of ~p", [GroupName])), + ct_logs:log("TEST INFO", "end_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), ok. @@ -1242,12 +1298,20 @@ report(What,Data) -> ok; {end_per_group,_} -> ok; + {ct_init_per_group,_} -> + ok; + {ct_end_per_group,_} -> + ok; {_,ok} -> add_to_stats(ok); {_,{skipped,{failed,{_,init_per_testcase,_}}}} -> add_to_stats(auto_skipped); {_,{skipped,{require_failed,_}}} -> add_to_stats(auto_skipped); + {_,{skipped,{timetrap_error,_}}} -> + add_to_stats(auto_skipped); + {_,{skipped,{invalid_time_format,_}}} -> + add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); {_,{SkipOrFail,_Reason}} -> @@ -1332,6 +1396,21 @@ add_data_dir(File,Config) when is_list(File) -> end. %%%----------------------------------------------------------------- +%%% @spec get_logopts() -> [LogOpt] +get_logopts() -> + case ct_util:get_testdata(logopts) of + undefined -> + []; + LogOpts -> + LogOpts + end. + +%%%----------------------------------------------------------------- +%%% @spec format_comment(Comment) -> HtmlComment +format_comment(Comment) -> + "<font color=\"green\">" ++ Comment ++ "</font>". + +%%%----------------------------------------------------------------- %%% @spec overview_html_header(TestName) -> Header overview_html_header(TestName) -> TestName1 = lists:flatten(io_lib:format("~p", [TestName])), diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 6a90441d53..c1523509a5 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -28,7 +28,7 @@ -module(ct_logs). --export([init/1,close/1,init_tc/0,end_tc/1]). +-export([init/1,close/2,init_tc/1,end_tc/1]). -export([get_log_dir/0,log/3,start_log/1,cont_log/2,end_log/0]). -export([set_stylesheet/2,clear_stylesheet/1]). -export([add_external_logs/1,add_link/3]). @@ -97,11 +97,11 @@ logdir_node_prefix() -> logdir_prefix()++"."++atom_to_list(node()). %%%----------------------------------------------------------------- -%%% @spec close(Info) -> ok +%%% @spec close(Info, StartDir) -> ok %%% %%% @doc Create index pages with test results and close the CT Log %%% (tool-internal use only). -close(Info) -> +close(Info, StartDir) -> make_last_run_index(), ct_event:notify(#event{name=stop_logging,node=node(),data=[]}), @@ -124,14 +124,29 @@ close(Info) -> ok; Error -> io:format("Warning! Cleanup failed: ~p~n", [Error]) - end; + end, + make_all_suites_index(stop), + make_all_runs_index(stop); true -> - file:set_cwd("..") - end, - - make_all_suites_index(stop), - make_all_runs_index(stop), - + file:set_cwd(".."), + make_all_suites_index(stop), + make_all_runs_index(stop), + case ct_util:get_profile_data(browser, StartDir) of + undefined -> + ok; + BrowserData -> + case {proplists:get_value(prog, BrowserData), + proplists:get_value(args, BrowserData), + proplists:get_value(page, BrowserData)} of + {Prog,Args,Page} when is_list(Args), + is_list(Page) -> + URL = "\"file://" ++ ?abs(Page) ++ "\"", + ct_util:open_url(Prog, Args, URL); + _ -> + ok + end + end + end, ok. %%%----------------------------------------------------------------- @@ -182,15 +197,14 @@ cast(Msg) -> ?MODULE ! Msg end. - %%%----------------------------------------------------------------- -%%% @spec init_tc() -> ok +%%% @spec init_tc(RefreshLog) -> ok %%% %%% @doc Test case initiation (tool-internal use only). %%% %%% <p>This function is called by ct_framework:init_tc/3</p> -init_tc() -> - call({init_tc,self(),group_leader()}), +init_tc(RefreshLog) -> + call({init_tc,self(),group_leader(),RefreshLog}), ok. %%%----------------------------------------------------------------- @@ -486,8 +500,8 @@ logger_loop(State) -> [Str,Args]), %% stop the testcase, we need %% to see the fault - exit(Pid,logging_failed), - ok; + exit(Pid,{log_printout_error,Str,Args}), + []; IoStr when IoList == [] -> [IoStr]; IoStr -> @@ -507,10 +521,15 @@ logger_loop(State) -> [begin io:format(Fd,Str,Args),io:nl(Fd) end || {Str,Args} <- List], logger_loop(State#logger_state{tc_groupleaders=TCGLs}) end; - {{init_tc,TCPid,GL},From} -> + {{init_tc,TCPid,GL,RefreshLog},From} -> print_style(GL, State#logger_state.stylesheet), set_evmgr_gl(GL), TCGLs = add_tc_gl(TCPid,GL,State), + if not RefreshLog -> + ok; + true -> + make_last_run_index(State#logger_state.start_time) + end, return(From,ok), logger_loop(State#logger_state{tc_groupleaders=TCGLs}); {{end_tc,TCPid},From} -> @@ -841,6 +860,7 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, "" end end, + CtRunDir = filename:dirname(filename:dirname(Link)), {Lbl,Timestamp,Node,AllInfo} = case All of {true,OldRuns} -> @@ -850,7 +870,6 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, _ -> NodeOrDate end, N = ["<TD ALIGN=right><FONT SIZE=-1>",Node1,"</FONT></TD>\n"], - CtRunDir = filename:dirname(filename:dirname(Link)), L = ["<TD ALIGN=center><FONT SIZE=-1><B>",Label,"</FONT></B></TD>\n"], T = ["<TD><FONT SIZE=-1>",timestamp(CtRunDir),"</FONT></TD>\n"], CtLogFile = filename:join(CtRunDir,?ct_log_name), @@ -869,7 +888,7 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, if NotBuilt == 0 -> ["<TD ALIGN=right>",integer_to_list(NotBuilt),"</TD>\n"]; true -> - ["<TD ALIGN=right><A HREF=\"",?ct_log_name,"\">", + ["<TD ALIGN=right><A HREF=\"",filename:join(CtRunDir,?ct_log_name),"\">", integer_to_list(NotBuilt),"</A></TD>\n"] end, FailStr = diff --git a/lib/common_test/src/ct_make.erl b/lib/common_test/src/ct_make.erl index 233e45248e..40e9e99f37 100644 --- a/lib/common_test/src/ct_make.erl +++ b/lib/common_test/src/ct_make.erl @@ -177,7 +177,7 @@ members([],_MakefileMods,I,Rest) -> {I,Rest}. -%% Any flags that are not recognixed as make flags are passed directly +%% Any flags that are not recognised as make flags are passed directly %% to the compiler. %% So for example make:all([load,debug_info]) will make everything %% with the debug_info flag and load it. diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 0715b8abf8..0a9bb5af67 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -33,7 +33,7 @@ %% Exported for VTS --export([run_make/3,do_run/3,tests/1,tests/2,tests/3]). +-export([run_make/3,do_run/4,tests/1,tests/2,tests/3]). %% Misc internal functions @@ -46,12 +46,14 @@ -define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)). -record(opts, {label, + profile, vts, shell, cover, coverspec, step, logdir, + logopts = [], config = [], event_handlers = [], ct_hooks = [], @@ -157,15 +159,19 @@ script_start(Args) -> end, stop_trace(Tracing), timer:sleep(1000), + io:nl(), Res. script_start1(Parent, Args) -> %% read general start flags Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args), + Profile = get_start_opt(profile, fun([Prof]) -> Prof end, Args), Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), + LogOpts = get_start_opt(logopts, fun(Os) -> [list_to_atom(O) || O <- Os] end, + [], Args), MultTT = get_start_opt(multiply_timetraps, fun([MT]) -> list_to_integer(MT) end, 1, Args), ScaleTT = get_start_opt(scale_timetraps, @@ -239,8 +245,10 @@ script_start1(Parent, Args) -> application:set_env(common_test, basic_html, true) end, - StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover, - logdir = LogDir, event_handlers = EvHandlers, + StartOpts = #opts{label = Label, profile = Profile, + vts = Vts, shell = Shell, cover = Cover, + logdir = LogDir, logopts = LogOpts, + event_handlers = EvHandlers, ct_hooks = CTHooks, enable_builtin_hooks = EnableBuiltinHooks, include = IncludeDirs, @@ -303,9 +311,15 @@ script_start2(StartOpts = #opts{vts = undefined, Label = choose_val(StartOpts#opts.label, SpecStartOpts#opts.label), + Profile = choose_val(StartOpts#opts.profile, + SpecStartOpts#opts.profile), + LogDir = choose_val(StartOpts#opts.logdir, SpecStartOpts#opts.logdir), + AllLogOpts = merge_vals([StartOpts#opts.logopts, + SpecStartOpts#opts.logopts]), + Cover = choose_val(StartOpts#opts.cover, SpecStartOpts#opts.cover), MultTT = choose_val(StartOpts#opts.multiply_timetraps, @@ -328,9 +342,11 @@ script_start2(StartOpts = #opts{vts = undefined, application:set_env(common_test, include, AllInclude), {TS,StartOpts#opts{label = Label, + profile = Profile, testspecs = Specs, cover = Cover, logdir = LogDir, + logopts = AllLogOpts, config = SpecStartOpts#opts.config, event_handlers = AllEvHs, ct_hooks = AllCTHooks, @@ -404,50 +420,72 @@ check_and_install_configfiles( end. script_start3(StartOpts, Args) -> - case proplists:get_value(dir, Args) of - [] -> + StartOpts1 = get_start_opt(step, + fun(Step) -> + StartOpts#opts{step = Step, + cover = undefined} + end, StartOpts, Args), + case {proplists:get_value(dir, Args), + proplists:get_value(suite, Args), + groups_and_cases(proplists:get_value(group, Args), + proplists:get_value(testcase, Args))} of + %% flag specified without data + {_,_,Error={error,_}} -> + Error; + {_,[],_} -> + {error,no_suite_specified}; + {[],_,_} -> {error,no_dir_specified}; - Dirs when is_list(Dirs) -> + + {Dirs,undefined,[]} when is_list(Dirs) -> script_start4(StartOpts#opts{tests = tests(Dirs)}, Args); - undefined -> - case proplists:get_value(suite, Args) of - [] -> - {error,no_suite_specified}; - Suites when is_list(Suites) -> - StartOpts1 = - get_start_opt(step, - fun(Step) -> - StartOpts#opts{step = Step, - cover = undefined} - end, StartOpts, Args), - DirMods = [suite_to_test(S) || S <- Suites], - case groups_and_cases(proplists:get_value(group, Args), - proplists:get_value(testcase, Args)) of - Error = {error,_} -> - Error; - [] when DirMods =/= [] -> - Ts = tests(DirMods), - script_start4(StartOpts1#opts{tests = Ts}, Args); - GroupsAndCases when length(DirMods) == 1 -> - Ts = tests(DirMods, GroupsAndCases), - script_start4(StartOpts1#opts{tests = Ts}, Args); - [_,_|_] when length(DirMods) > 1 -> - {error,multiple_suites_and_cases}; - _ -> - {error,incorrect_suite_option} - end; - undefined -> - if StartOpts#opts.vts ; StartOpts#opts.shell -> - script_start4(StartOpts#opts{tests = []}, Args); - true -> - script_usage(), - {error,incorrect_usage} - end + + {undefined,Suites,[]} when is_list(Suites) -> + Ts = tests([suite_to_test(S) || S <- Suites]), + script_start4(StartOpts1#opts{tests = Ts}, Args); + + {undefined,Suite,GsAndCs} when is_list(Suite) -> + case [suite_to_test(S) || S <- Suite] of + DirMods = [_] -> + Ts = tests(DirMods, GsAndCs), + script_start4(StartOpts1#opts{tests = Ts}, Args); + [_,_|_] -> + {error,multiple_suites_and_cases}; + _ -> + {error,incorrect_start_options} + end; + + {[_,_|_],Suite,[]} when is_list(Suite) -> + {error,multiple_dirs_and_suites}; + + {[Dir],Suite,GsAndCs} when is_list(Dir), is_list(Suite) -> + case [suite_to_test(Dir,S) || S <- Suite] of + DirMods when GsAndCs == [] -> + Ts = tests(DirMods), + script_start4(StartOpts1#opts{tests = Ts}, Args); + DirMods = [_] when GsAndCs /= [] -> + Ts = tests(DirMods, GsAndCs), + script_start4(StartOpts1#opts{tests = Ts}, Args); + [_,_|_] when GsAndCs /= [] -> + {error,multiple_suites_and_cases}; + _ -> + {error,incorrect_start_options} + end; + + {undefined,undefined,GsAndCs} when GsAndCs /= [] -> + {error,incorrect_start_options}; + + {undefined,undefined,_} -> + if StartOpts#opts.vts ; StartOpts#opts.shell -> + script_start4(StartOpts#opts{tests = []}, Args); + true -> + script_usage(), + {error,missing_start_options} end end. script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, - tests = Tests, logdir = LogDir}, _Args) -> + tests = Tests, logdir = LogDir, logopts = LogOpts}, _Args) -> ConfigFiles = lists:foldl(fun({ct_config_plain,CfgFiles}, AllFiles) when is_list(hd(CfgFiles)) -> @@ -458,16 +496,21 @@ script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, (_, AllFiles) -> AllFiles end, [], Config), - vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests); + vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), LogOpts, Tests); -script_start4(#opts{label = Label, shell = true, config = Config, +script_start4(#opts{label = Label, profile = Profile, + shell = true, config = Config, event_handlers = EvHandlers, ct_hooks = CTHooks, + logopts = LogOpts, enable_builtin_hooks = EnableBuiltinHooks, logdir = LogDir, testspecs = Specs}, _Args) -> %% label - used by ct_logs application:set_env(common_test, test_label, Label), + %% profile - used in ct_util + application:set_env(common_test, profile, Profile), + if Config == [] -> ok; true -> @@ -478,6 +521,7 @@ script_start4(#opts{label = Label, shell = true, config = Config, {enable_builtin_hooks,EnableBuiltinHooks}]) of ok -> ct_util:start(interactive, LogDir), + ct_util:set_testdata({logopts, LogOpts}), log_ts_names(Specs), io:nl(), ok; @@ -518,6 +562,7 @@ script_usage() -> "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" "\n\t[-dir TestDir1 TestDir2 .. TestDirN] |" "\n\t[-suite Suite [-case Case]]" + "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -534,8 +579,9 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" - "\n\t[-event_handler EvHandler1 and EvHandler2 .. EvHandlerN]" - "\n\t[-ct_hooks CTHook1 and CTHook2 .. CTHookN]" + "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -553,8 +599,9 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" - "\n\t[-event_handler EvHandler1 and EvHandler2 .. EvHandlerN]" - "\n\t[-ct_hooks CTHook1 and CTHook2 .. CTHookN]" + "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -632,6 +679,16 @@ run_test(StartOpt) when is_tuple(StartOpt) -> run_test([StartOpt]); run_test(StartOpts) when is_list(StartOpts) -> + CTPid = spawn(fun() -> run_test1(StartOpts) end), + Ref = monitor(process, CTPid), + receive + {'DOWN',Ref,process,CTPid,{user_error,Error}} -> + Error; + {'DOWN',Ref,process,CTPid,Other} -> + Other + end. + +run_test1(StartOpts) when is_list(StartOpts) -> case proplists:get_value(refresh_logs, StartOpts) of undefined -> Tracing = start_trace(StartOpts), @@ -640,7 +697,7 @@ run_test(StartOpts) when is_list(StartOpts) -> Res = case ct_repeat:loop_test(func, StartOpts) of false -> - case catch run_test1(StartOpts) of + case catch run_test2(StartOpts) of {'EXIT',Reason} -> file:set_cwd(Cwd), {error,Reason}; @@ -651,20 +708,27 @@ run_test(StartOpts) when is_list(StartOpts) -> Result end, stop_trace(Tracing), - Res; + exit(Res); RefreshDir -> refresh_logs(?abs(RefreshDir)), - ok + exit(done) end. -run_test1(StartOpts) -> +run_test2(StartOpts) -> %% label Label = get_start_opt(label, fun(Lbl) when is_list(Lbl) -> Lbl; (Lbl) when is_atom(Lbl) -> atom_to_list(Lbl) end, StartOpts), + %% profile + Profile = get_start_opt(profile, fun(Prof) when is_list(Prof) -> Prof; + (Prof) when is_atom(Prof) -> atom_to_list(Prof) + end, StartOpts), %% logdir LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end, StartOpts), + %% logopts + LogOpts = get_start_opt(logopts, value, [], StartOpts), + %% config & userconfig CfgFiles = ct_config:get_config_file_list(StartOpts), @@ -768,8 +832,9 @@ run_test1(StartOpts) -> %% stepped execution Step = get_start_opt(step, value, StartOpts), - Opts = #opts{label = Label, - cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, + Opts = #opts{label = Label, profile = Profile, + cover = Cover, step = Step, logdir = LogDir, + logopts = LogOpts, config = CfgFiles, event_handlers = EvHandlers, ct_hooks = CTHooks, enable_builtin_hooks = EnableBuiltinHooks, @@ -811,8 +876,12 @@ run_spec_file(Relaxed, SpecOpts = get_data_for_node(TS, node()), Label = choose_val(Opts#opts.label, SpecOpts#opts.label), + Profile = choose_val(Opts#opts.profile, + SpecOpts#opts.profile), LogDir = choose_val(Opts#opts.logdir, SpecOpts#opts.logdir), + AllLogOpts = merge_vals([Opts#opts.logopts, + SpecOpts#opts.logopts]), AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), Cover = choose_val(Opts#opts.cover, SpecOpts#opts.cover), @@ -833,8 +902,10 @@ run_spec_file(Relaxed, application:set_env(common_test, include, AllInclude), Opts1 = Opts#opts{label = Label, + profile = Profile, cover = Cover, logdir = which(logdir, LogDir), + logopts = AllLogOpts, config = AllConfig, event_handlers = AllEvHs, include = AllInclude, @@ -848,7 +919,6 @@ run_spec_file(Relaxed, case check_and_install_configfiles(AllConfig,Opts1#opts.logdir, Opts1) of ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); {error,GCFReason} -> @@ -921,67 +991,102 @@ run_dir(Opts = #opts{logdir = LogDir, ok -> ok; {error,IReason} -> exit(IReason) end, - case lists:keysearch(dir, 1, StartOpts) of - {value,{_,Dirs=[Dir|_]}} when not is_integer(Dir), - length(Dirs)>1 -> - %% multiple dirs (no suite) - reformat_result(catch do_run(tests(Dirs), [], Opts1, StartOpts)); - false -> % no dir - %% fun for converting suite name to {Dir,Mod} tuple - S2M = fun(S) when is_list(S) -> - {filename:dirname(S), - list_to_atom(filename:rootname(filename:basename(S)))}; - (A) -> - {".",A} - end, - case lists:keysearch(suite, 1, StartOpts) of - {value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) -> - {Dir,Mod} = S2M(Suite), - case groups_and_cases(proplists:get_value(group, StartOpts), - proplists:get_value(testcase, StartOpts)) of - Error = {error,_} -> - exit(Error); + case {proplists:get_value(dir, StartOpts), + proplists:get_value(suite, StartOpts), + groups_and_cases(proplists:get_value(group, StartOpts), + proplists:get_value(testcase, StartOpts))} of + %% flag specified without data + {_,_,Error={error,_}} -> + Error; + {_,[],_} -> + {error,no_suite_specified}; + {[],_,_} -> + {error,no_dir_specified}; + + {Dirs=[Hd|_],undefined,[]} when is_list(Dirs), not is_integer(Hd) -> + Dirs1 = [if is_atom(D) -> atom_to_list(D); + true -> D end || D <- Dirs], + reformat_result(catch do_run(tests(Dirs1), [], Opts1, StartOpts)); + + {Dir=[Hd|_],undefined,[]} when is_list(Dir) and is_integer(Hd) -> + reformat_result(catch do_run(tests(Dir), [], Opts1, StartOpts)); + + {Dir,undefined,[]} when is_atom(Dir) and (Dir /= undefined) -> + reformat_result(catch do_run(tests(atom_to_list(Dir)), + [], Opts1, StartOpts)); + + {undefined,Suites=[Hd|_],[]} when not is_integer(Hd) -> + Suites1 = [suite_to_test(S) || S <- Suites], + reformat_result(catch do_run(tests(Suites1), [], Opts1, StartOpts)); + + {undefined,Suite,[]} when is_atom(Suite) and + (Suite /= undefined) -> + {Dir,Mod} = suite_to_test(Suite), + reformat_result(catch do_run(tests(Dir, Mod), [], Opts1, StartOpts)); + + {undefined,Suite,GsAndCs} when is_atom(Suite) and + (Suite /= undefined) -> + {Dir,Mod} = suite_to_test(Suite), + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)); + + {undefined,[Hd,_|_],_GsAndCs} when not is_integer(Hd) -> + exit(multiple_suites_and_cases); + + {undefined,Suite=[Hd|Tl],GsAndCs} when is_integer(Hd) ; + (is_list(Hd) and (Tl == [])) ; + (is_atom(Hd) and (Tl == [])) -> + {Dir,Mod} = suite_to_test(Suite), + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)); + + {[Hd,_|_],_Suites,[]} when is_list(Hd) ; not is_integer(Hd) -> + exit(multiple_dirs_and_suites); + + {undefined,undefined,GsAndCs} when GsAndCs /= [] -> + exit(incorrect_start_options); + + {Dir,Suite,GsAndCs} when is_integer(hd(Dir)) ; + (is_atom(Dir) and (Dir /= undefined)) ; + ((length(Dir) == 1) and is_atom(hd(Dir))) ; + ((length(Dir) == 1) and is_list(hd(Dir))) -> + Dir1 = if is_atom(Dir) -> atom_to_list(Dir); + true -> Dir end, + if Suite == undefined -> + exit(incorrect_start_options); + + is_integer(hd(Suite)) ; + (is_atom(Suite) and (Suite /= undefined)) ; + ((length(Suite) == 1) and is_atom(hd(Suite))) ; + ((length(Suite) == 1) and is_list(hd(Suite))) -> + {Dir2,Mod} = suite_to_test(Dir1, Suite), + case GsAndCs of [] -> - reformat_result(catch do_run(tests(Dir, listify(Mod)), + reformat_result(catch do_run(tests(Dir2, Mod), [], Opts1, StartOpts)); - GsAndCs -> - reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + _ -> + reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), [], Opts1, StartOpts)) end; - {value,{_,Suites}} -> - reformat_result(catch do_run(tests(lists:map(S2M, Suites)), - [], Opts1, StartOpts)); - _ -> - exit(no_tests_specified) - end; - {value,{_,Dir}} -> - case lists:keysearch(suite, 1, StartOpts) of - {value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) -> - Mod = if is_atom(Suite) -> Suite; - true -> list_to_atom(Suite) - end, - case groups_and_cases(proplists:get_value(group, StartOpts), - proplists:get_value(testcase, StartOpts)) of - Error = {error,_} -> - exit(Error); - [] -> - reformat_result(catch do_run(tests(Dir, listify(Mod)), + + is_list(Suite) -> % multiple suites + case [suite_to_test(Dir1, S) || S <- Suite] of + [_,_|_] when GsAndCs /= [] -> + exit(multiple_suites_and_cases); + [{Dir2,Mod}] when GsAndCs /= [] -> + reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), [], Opts1, StartOpts)); - GsAndCs -> - reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + DirMods -> + reformat_result(catch do_run(tests(DirMods), [], Opts1, StartOpts)) - end; - {value,{_,Suites=[Suite|_]}} when is_list(Suite) -> - Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites), - reformat_result(catch do_run(tests(delistify(Dir), Mods), - [], Opts1, StartOpts)); - {value,{_,Suites}} -> - reformat_result(catch do_run(tests(delistify(Dir), Suites), - [], Opts1, StartOpts)); - false -> % no suite, only dir - reformat_result(catch do_run(tests(listify(Dir)), - [], Opts1, StartOpts)) - end + end + end; + + {undefined,undefined,[]} -> + exit(no_test_specified); + + {Dir,Suite,GsAndCs} -> + exit({incorrect_start_options,{Dir,Suite,GsAndCs}}) end. %%%----------------------------------------------------------------- @@ -992,19 +1097,38 @@ run_dir(Opts = #opts{logdir = LogDir, %%% the same as those used in test specification files. %%% @equiv ct:run_testspec/1 %%%----------------------------------------------------------------- - run_testspec(TestSpec) -> + CTPid = spawn(fun() -> run_testspec1(TestSpec) end), + Ref = monitor(process, CTPid), + receive + {'DOWN',Ref,process,CTPid,{user_error,Error}} -> + Error; + {'DOWN',Ref,process,CTPid,Other} -> + Other + end. + +run_testspec1(TestSpec) -> {ok,Cwd} = file:get_cwd(), io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]), - case catch run_testspec1(TestSpec) of + case catch run_testspec2(TestSpec) of {'EXIT',Reason} -> file:set_cwd(Cwd), - {error,Reason}; + exit({error,Reason}); Result -> - Result + exit(Result) end. -run_testspec1(TestSpec) -> +run_testspec2(File) when is_list(File), is_integer(hd(File)) -> + case file:read_file_info(File) of + {ok,_} -> + exit("Bad argument, " + "use ct:run_test([{spec," ++ File ++ "}])"); + _ -> + exit("Bad argument, list of tuples expected, " + "use ct:run_test/1 for test specification files") + end; + +run_testspec2(TestSpec) -> case catch ct_testspec:collect_tests_from_list(TestSpec, false) of {E,CTReason} when E == error ; E == 'EXIT' -> exit(CTReason); @@ -1035,7 +1159,9 @@ run_testspec1(TestSpec) -> end. get_data_for_node(#testspec{label = Labels, + profile = Profiles, logdir = LogDirs, + logopts = LogOptsList, cover = CoverFs, config = Cfgs, userconfig = UsrCfgs, @@ -1046,10 +1172,15 @@ get_data_for_node(#testspec{label = Labels, multiply_timetraps = MTs, scale_timetraps = STs}, Node) -> Label = proplists:get_value(Node, Labels), + Profile = proplists:get_value(Node, Profiles), LogDir = case proplists:get_value(Node, LogDirs) of undefined -> "."; Dir -> Dir end, + LogOpts = case proplists:get_value(Node, LogOptsList) of + undefined -> []; + LOs -> LOs + end, Cover = proplists:get_value(Node, CoverFs), MT = proplists:get_value(Node, MTs), ST = proplists:get_value(Node, STs), @@ -1059,7 +1190,9 @@ get_data_for_node(#testspec{label = Labels, FiltCTHooks = [Hook || {N,Hook} <- CTHooks, N==Node], Include = [I || {N,I} <- Incl, N==Node], #opts{label = Label, + profile = Profile, logdir = LogDir, + logopts = LogOpts, cover = Cover, config = ConfigFiles, event_handlers = EvHandlers, @@ -1141,8 +1274,24 @@ reformat_result({user_error,Reason}) -> reformat_result(Result) -> Result. -suite_to_test(Suite) -> - {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}. +suite_to_test(Suite) when is_atom(Suite) -> + suite_to_test(atom_to_list(Suite)); + +suite_to_test(Suite) when is_list(Suite) -> + {filename:dirname(Suite), + list_to_atom(filename:rootname(filename:basename(Suite)))}. + +suite_to_test(Dir, Suite) when is_atom(Suite) -> + suite_to_test(Dir, atom_to_list(Suite)); + +suite_to_test(Dir, Suite) when is_list(Suite) -> + case filename:dirname(Suite) of + "." -> + {Dir,list_to_atom(filename:rootname(Suite))}; + DirName -> % ignore Dir + File = filename:basename(Suite), + {DirName,list_to_atom(filename:rootname(File))} + end. groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and ((Cs == undefined) or (Cs == [])) -> @@ -1176,9 +1325,11 @@ tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) -> [{?testdir(TestDir,all),all,all} || TestDir <- TestDirs]. do_run(Tests, Misc) when is_list(Misc) -> - do_run(Tests, Misc, "."). + do_run(Tests, Misc, ".", []). -do_run(Tests, Misc, LogDir) when is_list(Misc) -> +do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), + is_list(LogDir), + is_list(LogOpts) -> Opts = case proplists:get_value(step, Misc) of undefined -> @@ -1193,11 +1344,10 @@ do_run(Tests, Misc, LogDir) when is_list(Misc) -> CoverFile -> Opts#opts{cover = CoverFile} end, - do_run(Tests, [], Opts1#opts{logdir = LogDir}, []). - -do_run(Tests, Skip, Opts, Args) -> - #opts{label = Label, cover = Cover} = Opts, + do_run(Tests, [], Opts1#opts{logdir = LogDir}, []); +do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> + #opts{label = Label, profile = Profile, cover = Cover} = Opts, %% label - used by ct_logs TestLabel = if Label == undefined -> undefined; @@ -1207,6 +1357,15 @@ do_run(Tests, Skip, Opts, Args) -> end, application:set_env(common_test, test_label, TestLabel), + %% profile - used in ct_util + TestProfile = + if Profile == undefined -> undefined; + is_atom(Profile) -> atom_to_list(Profile); + is_list(Profile) -> Profile; + true -> undefined + end, + application:set_env(common_test, profile, TestProfile), + case code:which(test_server) of non_existing -> exit({error,no_path_to_test_server}); @@ -1241,6 +1400,8 @@ do_run(Tests, Skip, Opts, Args) -> _Pid -> %% save stylesheet info ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), + %% save logopts + ct_util:set_testdata({logopts,Opts#opts.logopts}), %% enable silent connections case Opts#opts.silent_connections of [] -> @@ -2008,7 +2169,14 @@ maybe_interpret1(Suite, Cases, StepOpts) when is_list(Cases) -> maybe_interpret2(Suite, Cases, StepOpts) -> set_break_on_config(Suite, StepOpts), - [i:ib(Suite, Case, 1) || Case <- Cases], + [begin try i:ib(Suite, Case, 1) of + _ -> ok + catch + _:_Error -> + io:format(user, "Invalid breakpoint: ~w:~w/1~n", + [Suite,Case]) + end + end || Case <- Cases, is_atom(Case)], test_server_ctrl:multiply_timetraps(infinity), WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of true -> no_kill; @@ -2021,10 +2189,18 @@ maybe_interpret2(Suite, Cases, StepOpts) -> set_break_on_config(Suite, StepOpts) -> case lists:member(config, ensure_atom(StepOpts)) of true -> - i:ib(Suite, init_per_suite, 1), - i:ib(Suite, init_per_testcase, 2), - i:ib(Suite, end_per_testcase, 2), - i:ib(Suite, end_per_suite, 1); + SetBPIfExists = fun(F,A) -> + case erlang:function_exported(Suite, F, A) of + true -> i:ib(Suite, F, A); + false -> ok + end + end, + SetBPIfExists(init_per_suite, 1), + SetBPIfExists(init_per_group, 2), + SetBPIfExists(init_per_testcase, 2), + SetBPIfExists(end_per_testcase, 2), + SetBPIfExists(end_per_group, 2), + SetBPIfExists(end_per_suite, 1); false -> ok end. @@ -2078,6 +2254,15 @@ get_start_opt(Key, IfExists, Args) -> get_start_opt(Key, IfExists, undefined, Args). get_start_opt(Key, IfExists, IfNotExists, Args) -> + try try_get_start_opt(Key, IfExists, IfNotExists, Args) of + Result -> + Result + catch + error:_ -> + exit({user_error,{bad_argument,Key}}) + end. + +try_get_start_opt(Key, IfExists, IfNotExists, Args) -> case lists:keysearch(Key, 1, Args) of {value,{Key,Val}} when is_function(IfExists) -> IfExists(Val); @@ -2252,6 +2437,8 @@ opts2args(EnvStartOpts) -> end, EHs), [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), [{event_handler_init,lists:reverse(StrsR)}]; + ({logopts,LOs}) when is_list(LOs) -> + [{logopts,[atom_to_list(LO) || LO <- LOs]}]; ({ct_hooks,[]}) -> []; ({ct_hooks,CTHs}) when is_list(CTHs) -> @@ -2315,32 +2502,31 @@ is_suite(ModOrFile) when is_list(ModOrFile) -> end. get_all_testcases(Suite) -> - %%! this needs to be updated to handle testcase groups later!! - case catch Suite:all() of - {'EXIT',Why} -> - {error,Why}; - {skip,_} -> - []; - Cases -> - AllCases = - lists:foldl(fun({sequence,SeqName}, All) -> - case catch Suite:sequences() of - {'EXIT',_} -> - All; - Seqs -> - case proplists:get_value(SeqName, Seqs) of - undefined -> - All; - SeqCases -> - lists:reverse(SeqCases) ++ All - end - end; - (Case,All) -> - [Case|All] - end, [], Cases), - lists:reverse(AllCases) + try ct_framework:get_all_cases(Suite) of + {error,_Reason} = Error -> + Error; + SuiteCases -> + Cases = [C || {_S,C} <- SuiteCases], + try Suite:sequences() of + [] -> + Cases; + Seqs -> + TCs1 = lists:flatten([TCs || {_,TCs} <- Seqs]), + lists:reverse( + lists:foldl(fun(TC, Acc) -> + case lists:member(TC, Acc) of + true -> Acc; + false -> [TC | Acc] + end + end, [], Cases ++ TCs1)) + catch + _:_ -> + Cases + end + catch + _:Error -> + {error,Error} end. - %% Internal tracing support. If {ct_trace,TraceSpec} is present, the %% TraceSpec file will be consulted and dbg used to trace function @@ -2361,8 +2547,8 @@ start_trace(Args) -> false end; {_,Error} -> - io:format("Warning! Tracing not started. Reason: ~p~n~n", - [Error]), + io:format("Warning! Tracing not started. Reason: ~s~n~n", + [file:format_error(Error)]), false end; false -> diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index c6f5fd7df4..71a784870c 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -245,7 +245,6 @@ cmdf(Connection,CmdFormat,Args) -> %%% Data = [string()] %%% @doc Send a telnet command and wait for prompt %%% (uses a format string and list of arguments to build the command). -%%%----------------------------------------------------------------- cmdf(Connection,CmdFormat,Args,Timeout) when is_list(Args) -> Cmd = lists:flatten(io_lib:format(CmdFormat,Args)), cmd(Connection,Cmd,Timeout). @@ -360,15 +359,15 @@ expect(Connection,Patterns) -> %%% will also be a <code>HaltReason</code> returned.</p> %%% %%% <p><underline>Examples:</underline><br/> -%%% <code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}], -%%% [sequence,{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match +%%% <code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}],</code> +%%% <code>[sequence,{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match %%% "ABC" first and then "XYZ", but if "NNN" appears the function will %%% return <code>{error,{nnn,["NNN"]}}</code>. If both "ABC" and "XYZ" %%% are matched, the function will return %%% <code>{ok,[AbcMatch,XyzMatch]}</code>.</p> %%% -%%% <p><code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}], -%%% [{repeat,2},{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match +%%% <p><code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}],</code> +%%% <code>[{repeat,2},{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match %%% "ABC" or "XYZ" twice. If "NNN" appears the function will return %%% with <code>HaltReason = {nnn,["NNN"]}</code>.</p> %%% diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 96971ccfc7..317910d5c8 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -249,11 +249,15 @@ collect_tests_from_file1([Spec|Specs],TestSpec,Relaxed) -> SpecDir = filename:dirname(filename:absname(Spec)), case file:consult(Spec) of {ok,Terms} -> - TestSpec1 = collect_tests(Terms,TestSpec#testspec{spec_dir=SpecDir}, + TestSpec1 = collect_tests(Terms, + TestSpec#testspec{spec_dir=SpecDir}, Relaxed), collect_tests_from_file1(Specs,TestSpec1,Relaxed); {error,Reason} -> - throw({error,{Spec,Reason}}) + ReasonStr = + lists:flatten(io_lib:format("~s", + [file:format_error(Reason)])), + throw({error,{Spec,ReasonStr}}) end; collect_tests_from_file1([],TS=#testspec{config=Cfgs,event_handler=EvHs, include=Incl,tests=Tests},_) -> @@ -481,6 +485,26 @@ add_tests([{logdir,Node,Dir}|Ts],Spec) -> add_tests([{logdir,Dir}|Ts],Spec) -> add_tests([{logdir,all_nodes,Dir}|Ts],Spec); +%% --- logopts --- +add_tests([{logopts,all_nodes,Opts}|Ts],Spec) -> + LogOpts = Spec#testspec.logopts, + Tests = [{logopts,N,Opts} || + N <- list_nodes(Spec), + lists:keymember(ref2node(N,Spec#testspec.nodes),1, + LogOpts) == false], + add_tests(Tests++Ts,Spec); +add_tests([{logopts,Nodes,Opts}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,logopts,[Opts],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{logopts,Node,Opts}|Ts],Spec) -> + LogOpts = Spec#testspec.logopts, + LogOpts1 = [{ref2node(Node,Spec#testspec.nodes),Opts} | + lists:keydelete(ref2node(Node,Spec#testspec.nodes), + 1,LogOpts)], + add_tests(Ts,Spec#testspec{logopts=LogOpts1}); +add_tests([{logopts,Opts}|Ts],Spec) -> + add_tests([{logopts,all_nodes,Opts}|Ts],Spec); + %% --- label --- add_tests([{label,all_nodes,Lbl}|Ts],Spec) -> Labels = Spec#testspec.label, @@ -1101,6 +1125,8 @@ valid_terms() -> {merge_tests,1}, {logdir,2}, {logdir,3}, + {logopts,2}, + {logopts,3}, {label,2}, {label,3}, {event_handler,2}, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index b3e345b4e5..3b6ad6f98d 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -47,7 +47,7 @@ -export([get_mode/0, create_table/3, read_opts/0]). --export([set_cwd/1, reset_cwd/0]). +-export([set_cwd/1, reset_cwd/0, get_start_dir/0]). -export([parse_table/1]). @@ -61,6 +61,9 @@ -export([warn_duplicates/1]). +-export([get_profile_data/0, get_profile_data/1, + get_profile_data/2, open_url/3]). + -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -121,13 +124,15 @@ do_start(Parent,Mode,LogDir) -> ok -> ok; E -> exit(E) end, + DoExit = fun(Reason) -> file:set_cwd(StartDir), exit(Reason) end, Opts = case read_opts() of {ok,Opts1} -> Opts1; Error -> Parent ! {self(),Error}, - exit(Error) + DoExit(Error) end, + %% start an event manager (if not already started by master) case ct_event:start_link() of {error,{already_started,_}} -> @@ -140,16 +145,23 @@ do_start(Parent,Mode,LogDir) -> ct_event:add_handler([{vts,VtsPid}]) end end, + %% start ct_config server - ct_config:start(Mode), + try ct_config:start(Mode) of + _ -> ok + catch + _Class:CfgError -> + DoExit(CfgError) + end, + %% add user event handlers case lists:keysearch(event_handler,1,Opts) of {value,{_,Handlers}} -> Add = fun({H,Args}) -> case catch gen_event:add_handler(?CT_EVMGR_REF,H,Args) of ok -> ok; - {'EXIT',Why} -> exit(Why); - Other -> exit({event_handler,Other}) + {'EXIT',Why} -> DoExit(Why); + Other -> DoExit({event_handler,Other}) end end, case catch lists:foreach(Add,Handlers) of @@ -168,10 +180,15 @@ do_start(Parent,Mode,LogDir) -> data={StartTime, lists:flatten(TestLogDir)}}), %% Initialize ct_hooks - case catch ct_hooks:init(Opts) of + try ct_hooks:init(Opts) of ok -> Parent ! {self(),started}; - {_,CTHReason} -> + {fail,CTHReason} -> + ct_logs:tc_print('Suite Callback',CTHReason,[]), + self() ! {{stop,{self(),{user_error,CTHReason}}}, + {Parent,make_ref()}} + catch + _:CTHReason -> ct_logs:tc_print('Suite Callback',CTHReason,[]), self() ! {{stop,{self(),{user_error,CTHReason}}}, {Parent,make_ref()}} @@ -243,6 +260,9 @@ set_cwd(Dir) -> reset_cwd() -> call(reset_cwd). +get_start_dir() -> + call(get_start_dir). + loop(Mode,TestData,StartDir) -> receive {update_last_run_index,From} -> @@ -319,6 +339,9 @@ loop(Mode,TestData,StartDir) -> {reset_cwd,From} -> return(From,file:set_cwd(StartDir)), loop(From,TestData,StartDir); + {get_start_dir,From} -> + return(From,StartDir), + loop(From,TestData,StartDir); {{stop,Info},From} -> Time = calendar:local_time(), ct_event:sync_notify(#event{name=test_done, @@ -332,7 +355,7 @@ loop(Mode,TestData,StartDir) -> ets:delete(?conn_table), ets:delete(?board_table), ets:delete(?suite_table), - ct_logs:close(Info), + ct_logs:close(Info, StartDir), ct_event:stop(), ct_config:stop(), file:set_cwd(StartDir), @@ -727,6 +750,79 @@ warn_duplicates(Suites) -> lists:foreach(Warn, Suites), ok. +%%%----------------------------------------------------------------- +%%% @spec +%%% +%%% @doc +get_profile_data() -> + get_profile_data(all). + +get_profile_data(KeyOrStartDir) -> + if is_atom(KeyOrStartDir) -> + get_profile_data(KeyOrStartDir, get_start_dir()); + is_list(KeyOrStartDir) -> + get_profile_data(all, KeyOrStartDir) + end. + +get_profile_data(Key, StartDir) -> + Profile = case application:get_env(common_test, profile) of + {ok,undefined} -> default; + {ok,Prof} -> Prof; + _ -> default + end, + get_profile_data(Profile, Key, StartDir). + +get_profile_data(Profile, Key, StartDir) -> + File = case Profile of + default -> + ?ct_profile_file; + _ when is_list(Profile) -> + ?ct_profile_file ++ "." ++ Profile; + _ when is_atom(Profile) -> + ?ct_profile_file ++ "." ++ atom_to_list(Profile) + end, + FullNameWD = filename:join(StartDir, File), + {WhichFile,Result} = + case file:consult(FullNameWD) of + {error,enoent} -> + case init:get_argument(home) of + {ok,[[HomeDir]]} -> + FullNameHome = filename:join(HomeDir, File), + {FullNameHome,file:consult(FullNameHome)}; + _ -> + {File,{error,enoent}} + end; + Consulted -> + {FullNameWD,Consulted} + end, + case Result of + {error,enoent} when Profile /= default -> + io:format(user, "~nERROR! Missing profile file ~p~n", [File]), + undefined; + {error,enoent} when Profile == default -> + undefined; + {error,Reason} -> + io:format(user,"~nERROR! Error in profile file ~p: ~p~n", + [WhichFile,Reason]), + undefined; + {ok,Data} -> + Data1 = case Data of + [List] when is_list(List) -> + List; + _ when is_list(Data) -> + Data; + _ -> + io:format(user, + "~nERROR! Invalid profile data in ~p~n", + [WhichFile]), + [] + end, + if Key == all -> + Data1; + true -> + proplists:get_value(Key, Data) + end + end. %%%----------------------------------------------------------------- %%% Internal functions @@ -799,3 +895,28 @@ abs_name2([H|T],Acc) -> abs_name2(T,[H|Acc]); abs_name2([],Acc) -> filename:join(lists:reverse(Acc)). + +open_url(iexplore, Args, URL) -> + {ok,R} = win32reg:open([read]), + ok = win32reg:change_key(R,"applications\\iexplore.exe\\shell\\open\\command"), + case win32reg:values(R) of + {ok, Paths} -> + Path = proplists:get_value(default, Paths), + [Cmd | _] = string:tokens(Path, "%"), + Cmd1 = Cmd ++ " " ++ Args ++ " " ++ URL, + io:format(user, "~nOpening ~s with command:~n ~s~n", [URL,Cmd1]), + open_port({spawn,Cmd1}, []); + _ -> + io:format("~nNo path to iexplore.exe~n",[]) + end, + win32reg:close(R), + ok; + +open_url(Prog, Args, URL) -> + ProgStr = if is_atom(Prog) -> atom_to_list(Prog); + is_list(Prog) -> Prog + end, + Cmd = ProgStr ++ " " ++ Args ++ " " ++ URL, + io:format(user, "~nOpening ~s with command:~n ~s~n", [URL,Cmd]), + open_port({spawn,Cmd},[]), + ok. diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index dbe9d9b4e6..bde832811a 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -31,7 +31,9 @@ nodes=[], init=[], label=[], + profile=[], logdir=["."], + logopts=[], cover=[], config=[], userconfig=[], @@ -59,3 +61,5 @@ -define(missing_suites_info, "missing_suites.info"). -define(ct_config_txt, ct_config_plain). + +-define(ct_profile_file, ".common_test"). diff --git a/lib/common_test/src/vts.erl b/lib/common_test/src/vts.erl index f0bf090804..cc8a932887 100644 --- a/lib/common_test/src/vts.erl +++ b/lib/common_test/src/vts.erl @@ -20,7 +20,7 @@ -module(vts). -export([start/0, - init_data/4, + init_data/5, stop/0, report/2]). @@ -32,6 +32,7 @@ menu_frame/2, welcome_frame/2, config_frame/2, + browse_config_file/2, add_config_file/2, remove_config_file/2, run_frame/2, @@ -56,7 +57,7 @@ -record(state,{tests=[],config=[],event_handler=[],test_runner, running=0,reload_results=false,start_dir,current_log_dir, - total=0,ok=0,fail=0,skip=0,testruns=[]}). + logopts=[],total=0,ok=0,fail=0,skip=0,testruns=[]}). %%%----------------------------------------------------------------- @@ -65,8 +66,8 @@ start() -> webtool:start(), webtool:start_tools([],"app=vts"). -init_data(ConfigFiles,EvHandlers,LogDir,Tests) -> - call({init_data,ConfigFiles,EvHandlers,LogDir,Tests}). +init_data(ConfigFiles,EvHandlers,LogDir,LogOpts,Tests) -> + call({init_data,ConfigFiles,EvHandlers,LogDir,LogOpts,Tests}). stop() -> webtool:stop_tools([],"app=vts"), @@ -119,6 +120,8 @@ menu_frame(_Env,_Input) -> call(menu_frame). config_frame(_Env,_Input) -> call(config_frame). +browse_config_file(_Env,Input) -> + call({browse_config_file,Input}). add_config_file(_Env,Input) -> call({add_config_file,Input}). remove_config_file(_Env,Input) -> @@ -160,10 +163,11 @@ init(Parent) -> loop(State) -> receive - {{init_data,Config,EvHandlers,LogDir,Tests},From} -> + {{init_data,Config,EvHandlers,LogDir,LogOpts,Tests},From} -> %% ct:pal("State#state.current_log_dir=~p", [State#state.current_log_dir]), NewState = State#state{config=Config,event_handler=EvHandlers, - current_log_dir=LogDir,tests=Tests}, + current_log_dir=LogDir, + logopts=LogOpts,tests=Tests}, ct_install(NewState), return(From,ok), loop(NewState); @@ -182,6 +186,9 @@ loop(State) -> {config_frame,From} -> return(From,config_frame1(State)), loop(State); + {{browse_config_file,_Input},From} -> + return(From,ok), + loop(State); {{add_config_file,Input},From} -> {Return,State1} = add_config_file1(Input,State), ct_install(State1), @@ -241,10 +248,12 @@ loop(State) -> return(From,ok); {'EXIT',Pid,Reason} -> case State#state.test_runner of - Pid -> io:format("ERROR: test runner crashed: ~p\n",[Reason]); - _ -> ignore - end, - loop(State); + Pid -> + io:format("Test run error: ~p\n",[Reason]), + loop(State); + _ -> + loop(State) + end; {{test_info,_Type,_Data},From} -> return(From,ok), loop(State) @@ -270,10 +279,11 @@ return({To,Ref},Result) -> To ! {Ref, Result}. -run_test1(State=#state{tests=Tests,current_log_dir=LogDir}) -> +run_test1(State=#state{tests=Tests,current_log_dir=LogDir, + logopts=LogOpts}) -> Self=self(), RunTest = fun() -> - case ct_run:do_run(Tests,[],LogDir) of + case ct_run:do_run(Tests,[],LogDir,LogOpts) of {error,_Reason} -> aborted(); _ -> @@ -282,17 +292,18 @@ run_test1(State=#state{tests=Tests,current_log_dir=LogDir}) -> unlink(Self) end, Pid = spawn_link(RunTest), - Total = + {Total,Tests1} = receive {{test_info,start_info,{_,_,Cases}},From} -> return(From,ok), - Cases; + {Cases,Tests}; EXIT = {'EXIT',_,_} -> - self() ! EXIT + self() ! EXIT, + {0,[]} after 30000 -> - 0 + {0,[]} end, - State#state{test_runner=Pid,running=length(Tests), + State#state{test_runner=Pid,running=length(Tests1), total=Total,ok=0,fail=0,skip=0,testruns=[]}. @@ -356,22 +367,32 @@ config_frame1(State) -> config_body(State) -> Entry = [input("TYPE=file NAME=browse SIZE=40"), input("TYPE=hidden NAME=file")], + BrowseForm = + form( + "NAME=read_file_form METHOD=post ACTION=\"./browse_config_file\"", + table( + "BORDER=0", + [tr(td("1. Locate config file")), + tr(td(Entry))])), AddForm = form( - "NAME=read_file_form METHOD=post ACTION=\"./add_config_file\"", + "NAME=add_file_form METHOD=post ACTION=\"./add_config_file\"", table( "BORDER=0", - [tr( - [td(Entry), + [tr(td("2. Paste full config file name here")), + tr( + [td(input("TYPE=text NAME=file SIZE=40")), td("ALIGN=center", input("TYPE=submit onClick=\"file.value=browse.value;\"" " VALUE=\"Add\""))])])), + {Text,RemoveForm} = case State#state.config of [] -> - T = "To be able to run any tests, one or more configuration " - "files must be added. Enter the name of the configuration " - "file below and click the \"Add\" button.", + T = "Before running the tests, one or more configuration " + "files may be added. Locate the config file, copy its " + "full name, paste this into the text field below, then " + "click the \"Add\" button.", R = "", {T,R}; Files -> @@ -394,20 +415,24 @@ config_body(State) -> input("TYPE=submit VALUE=\"Remove\"")))])), {T,R} end, - + [h1("ALIGN=center","Config"), table( - "WIDTH=600 ALIGN=center CELLPADDING=5", + "WIDTH=450 ALIGN=center CELLPADDING=5", [tr(td(["BGCOLOR=",?INFO_BG_COLOR],Text)), - tr(td("ALIGN=center",AddForm)), - tr(td("ALIGN=center",RemoveForm))])]. - + tr(td("")), + tr(td("")), + tr(td("ALIGN=left",BrowseForm)), + tr(td("ALIGN=left",AddForm)), + tr(td("ALIGN=left",RemoveForm))])]. add_config_file1(Input,State) -> State1 = case get_input_data(Input,"file") of - "" -> State; - File -> State#state{config=[File|State#state.config]} + "" -> + State; + File -> + State#state{config=[File|State#state.config]} end, Return = config_frame1(State1), {Return,State1}. @@ -427,10 +452,17 @@ run_body(#state{running=Running}) when Running>0 -> [h1("ALIGN=center","Run Test"), p(["Test are ongoing: ",href("./result_frameset","Results")])]; run_body(State) -> - ConfigList = ul([li(File) || File <- State#state.config]), + ConfigList = + case State#state.config of + [] -> + ul(["none"]); + CfgFiles -> + ul([li(File) || File <- CfgFiles]) + end, ConfigFiles = [h3("Config Files"), ConfigList], - + {ok,CWD} = file:get_cwd(), + CurrWD = [h3("Current Working Directory"), ul(CWD)], AddDirForm = form( "NAME=add_dir_form METHOD=post ACTION=\"./add_test_dir\"", @@ -442,7 +474,6 @@ run_body(State) -> td("ALIGN=center", input("TYPE=submit onClick=\"dir.value=browse.value;\"" " VALUE=\"Add Test Dir\""))])])), - {LoadedTestsTable,Submit} = case create_testdir_entries(State#state.tests,1) of [] -> {"",""}; @@ -454,22 +485,20 @@ run_body(State) -> {table("CELLPADDING=5",[Heading,TestDirs]), submit_button()} end, - - %% It should be ok to have no config-file! Body = - %% case State#state.config of %% [] -> %% p("ALIGN=center", - %% href("./config_frame","Please select one or - %% more config files")); %% _ -> table( - "WIDTH=100%", - [tr(td(ConfigFiles)), + "WIDTH=450 ALIGN=center", + [tr(td("")), + tr(td("")), + tr(td(ConfigFiles)), + tr(td("")), + tr(td(CurrWD)), tr(td("")), tr(td(AddDirForm)), tr(td("")), tr(td(LoadedTestsTable)), - tr(td(Submit))]), - %% end, - + tr(td(Submit)) + ]), [h1("ALIGN=center","Run Test"), Body]. create_testdir_entries([{Dir,Suite,Case}|Tests],N) -> @@ -556,18 +585,17 @@ options([Element|Elements],Selected,N,Func) -> options([],_Selected,_N,_Func) -> []. -add_test_dir1(Input,State) -> +add_test_dir1(Input, State) -> State1 = case get_input_data(Input,"dir") of "" -> State; Dir0 -> Dir = case ct_util:is_test_dir(Dir0) of - true -> - Dir0; - false -> filename:join(Dir0,"test") + true -> Dir0; + false -> ct_util:get_testdir(Dir0, all) end, case filelib:is_dir(Dir) of - true -> + true -> Test = ct_run:tests(Dir), State#state{tests=State#state.tests++Test}; false -> @@ -577,8 +605,6 @@ add_test_dir1(Input,State) -> Return = run_frame1(State1), {Return,State1}. - - remove_test_dir1(Input,State) -> N = list_to_integer(get_input_data(Input,"dir")), State1 = State#state{tests=delete_test(N,State#state.tests)}, @@ -641,6 +667,9 @@ result_frameset2(State) -> "./redirect_to_result_log_frame"; {_Dir,0} -> filename:join(["/log_dir","index.html"]); + {_Dir,_} when State#state.testruns == [] -> + %% crash before first test + "./no_result_log_frame"; {_Dir,_} -> {_,CurrentLog} = hd(State#state.testruns), CurrentLog @@ -749,6 +778,8 @@ report1(tc_done,{_Suite,_Case,{skipped,_Reason}},State) -> State#state{skip=State#state.skip+1}; report1(tc_user_skip,{_Suite,_Case,_Reason},State) -> State#state{skip=State#state.skip+1}; +report1(tc_auto_skip,{_Suite,_Case,_Reason},State) -> + State#state{skip=State#state.skip+1}; report1(loginfo,_,State) -> State. @@ -850,6 +881,8 @@ h2(Text) -> ["<H2>",Text,"</H2>\n"]. h3(Text) -> ["<H3>",Text,"</H3>\n"]. +%%h4(Text) -> +%% ["<H4>",Text,"</H4>\n"]. font(Args,Text) -> ["<FONT ",Args,">\n",Text,"\n</FONT>\n"]. p(Text) -> diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 836443009f..c1a455c6d8 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -60,7 +60,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, - timetrap_normal, timetrap_extended]. + timetrap_normal, timetrap_extended, timetrap_parallel, + timetrap_fun]. groups() -> []. @@ -228,6 +229,28 @@ timetrap_parallel(Config) when is_list(Config) -> ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- +%%% +timetrap_fun(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_4_SUITE"), + Join(DataDir, "timetrap_5_SUITE"), + Join(DataDir, "timetrap_6_SUITE"), + Join(DataDir, "timetrap_7_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_fun, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(timetrap_fun), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -260,7 +283,7 @@ test_events(cfg_error) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{14,14,43}}, + {?eh,start_info,{14,14,45}}, {?eh,tc_start,{cfg_error_1_SUITE,init_per_suite}}, {?eh,tc_done, @@ -517,13 +540,18 @@ test_events(cfg_error) -> {test_server,my_apply,3}, {test_server,do_end_per_testcase,4}, {test_server,run_test_case_eval1,6}, - {test_server,run_test_case_eval,8}]}}}}}}, + {test_server,run_test_case_eval,9}]}}}}}}, {?eh,test_stats,{12,3,{0,18}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc14}}, {?eh,tc_done, {cfg_error_9_SUITE,tc14,{failed,{error,tc14_should_be_failed}}}}, - {?eh,test_stats,{12,4,{0,18}}}, - + {?eh,tc_start,{cfg_error_9_SUITE,tc15}}, + {?eh,tc_done, + {cfg_error_9_SUITE,tc15,{failed,{error,this_error_must_show}}}}, + {?eh,tc_start,{cfg_error_9_SUITE,tc16}}, + {?eh,tc_done, + {cfg_error_9_SUITE,tc16,{failed,{error,this_error_must_show}}}}, + {?eh,test_stats,{12,6,{0,18}}}, {?eh,tc_start,{cfg_error_9_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_9_SUITE,end_per_suite,ok}}, @@ -533,7 +561,7 @@ test_events(cfg_error) -> {?eh,tc_auto_skip,{cfg_error_10_SUITE,tc1, {failed,{cfg_error_10_SUITE,init_per_suite, {failed,fail_init_per_suite}}}}}, - {?eh,test_stats,{12,4,{0,19}}}, + {?eh,test_stats,{12,6,{0,19}}}, {?eh,tc_auto_skip,{cfg_error_10_SUITE,end_per_suite, {failed,{cfg_error_10_SUITE,init_per_suite, {failed,fail_init_per_suite}}}}}, @@ -542,40 +570,40 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, {?eh,tc_done,{cfg_error_11_SUITE,tc1, {skipped,{config_name_already_in_use,[dummy0]}}}}, - {?eh,test_stats,{12,4,{1,19}}}, + {?eh,test_stats,{12,6,{1,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, - {?eh,test_stats,{13,4,{1,19}}}, + {?eh,test_stats,{13,6,{1,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, {?eh,tc_done,{cfg_error_12_SUITE,tc1,{failed,{timetrap_timeout,500}}}}, - {?eh,test_stats,{13,5,{1,19}}}, + {?eh,test_stats,{13,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,5,{1,19}}}, + {?eh,test_stats,{14,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,5,{1,19}}}, + {?eh,test_stats,{15,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,5,{1,19}}}, + {?eh,test_stats,{16,7,{1,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,5,{1,19}}}, + {?eh,test_stats,{17,7,{1,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,5,{1,19}}}, + {?eh,test_stats,{18,7,{1,19}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, @@ -603,7 +631,7 @@ test_events(lib_error) -> {?eh,test_stats,{0,2,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,lines_hang}}, {?eh,tc_done, - {lib_lines,do_hang,{failed,{timetrap_timeout,3000}}}}, + {lib_error_1_SUITE,lines_hang,{failed,{timetrap_timeout,3000}}}}, {?eh,test_stats,{0,3,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,lines_throw}}, {?eh,tc_done, @@ -752,7 +780,7 @@ test_events(timetrap_parallel) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,7}}, + {?eh,start_info,{1,1,8}}, {?eh,tc_done,{timetrap_3_SUITE,init_per_suite,ok}}, {parallel, [{?eh,tc_start, @@ -764,9 +792,12 @@ test_events(timetrap_parallel) -> {?eh,tc_start,{timetrap_3_SUITE,tc2}}, {?eh,tc_start,{timetrap_3_SUITE,tc3}}, {?eh,tc_start,{timetrap_3_SUITE,tc4}}, + {?eh,tc_start,{timetrap_3_SUITE,tc5}}, {?eh,tc_start,{timetrap_3_SUITE,tc6}}, {?eh,tc_start,{timetrap_3_SUITE,tc7}}, {?eh,tc_done, + {timetrap_3_SUITE,tc5,ok}}, + {?eh,tc_done, {timetrap_3_SUITE,tc1,{failed,{timetrap_timeout,500}}}}, {?eh,tc_done, {timetrap_3_SUITE,tc2,{failed,{timetrap_timeout,1000}}}}, @@ -780,11 +811,90 @@ test_events(timetrap_parallel) -> {timetrap_3_SUITE,tc4,{failed,{timetrap_timeout,2000}}}}, {?eh,tc_done, {timetrap_3_SUITE,tc3,{failed,{timetrap_timeout,3000}}}}, - {?eh,test_stats,{0,7,{0,0}}}, + {?eh,test_stats,{1,7,{0,0}}}, {?eh,tc_start, {timetrap_3_SUITE,{end_per_group,g1,[parallel]}}}, {?eh,tc_done, {timetrap_3_SUITE,{end_per_group,g1,[parallel]},ok}}]}, {?eh,tc_done,{timetrap_3_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]}]. + {?eh,stop_logging,[]}]; + +test_events(timetrap_fun) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,start_info,{4,4,17}}, + {?eh,tc_done,{timetrap_4_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_4_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc0,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_4_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc1,{failed,{timetrap_timeout,2000}}}}, + {?eh,tc_start,{timetrap_4_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc2,{failed,{timetrap_timeout,500}}}}, + {?eh,tc_start,{timetrap_4_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc3,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_done,{timetrap_4_SUITE,end_per_suite,ok}}, + + {?eh,tc_done,{timetrap_5_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_5_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc0,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc1,{skipped,{timetrap_error,kaboom}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc2,{skipped,{timetrap_error,kaboom}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc3, + {skipped,{invalid_time_format,{timetrap_utils,timetrap_val,[5000]}}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc4}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc4,{skipped,{invalid_time_format,'_'}}}}, + {?eh,test_stats,{0,5,{0,4}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc5}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc5,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc6}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc6,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc7}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc7,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,8,{0,4}}}, + {?eh,tc_done,{timetrap_5_SUITE,end_per_suite,ok}}, + + {?eh,tc_start,{timetrap_6_SUITE,init_per_suite}}, + {?eh,tc_done, + {timetrap_6_SUITE,init_per_suite,{skipped,{timetrap_error,kaboom}}}}, + {?eh,tc_auto_skip, + {timetrap_6_SUITE,tc0,{fw_auto_skip,{timetrap_error,kaboom}}}}, + {?eh,test_stats,{0,8,{0,5}}}, + {?eh,tc_auto_skip, + {timetrap_6_SUITE,end_per_suite,{fw_auto_skip,{timetrap_error,kaboom}}}}, + + {?eh,tc_done,{timetrap_7_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_7_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc0,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_7_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc1,{failed,{timetrap_timeout,2000}}}}, + {?eh,tc_start,{timetrap_7_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc2,{failed,{timetrap_timeout,500}}}}, + {?eh,tc_start,{timetrap_7_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc3,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,12,{0,5}}}, + {?eh,tc_done,{timetrap_7_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl index c4e0d72948..f292985c0c 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl @@ -76,7 +76,7 @@ init_per_testcase(tc1, Config) -> Config; init_per_testcase(tc2, Config) -> ct:comment("init_per_testcase(tc2) timeout"), - timer:sleep(5000), + ct:sleep(5000), Config; init_per_testcase(tc3, Config) -> badmatch = ?config(void, Config), @@ -96,22 +96,20 @@ init_per_testcase(_, Config) -> %%-------------------------------------------------------------------- end_per_testcase(tc11, _Config) -> ct:comment("A warning should be printed"), - exit(warning_should_be_printed), - done; + exit(warning_should_be_printed); end_per_testcase(tc12, _Config) -> ct:comment("A warning should be printed"), - timer:sleep(5000), - done; + ct:sleep(5000); end_per_testcase(tc13, Config) -> ct:comment("A warning should be printed"), - badmatch = ?config(void, Config), - done; + badmatch = ?config(void, Config); end_per_testcase(tc14, Config) -> ok = ?config(tc_status, Config), {fail,tc14_should_be_failed}; end_per_testcase(tc15, Config) -> - {failed,byebye} = ?config(tc_status, Config), - ok; + exit(kaboom); +end_per_testcase(tc16, Config) -> + ct:sleep(5000); end_per_testcase(_TestCase, _Config) -> done. @@ -139,7 +137,7 @@ groups() -> %%-------------------------------------------------------------------- all() -> [tc1,tc2,tc3,tc4,tc5,tc6,tc7, - tc11,tc12,tc13,tc14]. + tc11,tc12,tc13,tc14,tc15,tc16]. tc1(_) -> fini. @@ -189,4 +187,6 @@ tc14(_) -> ct:comment("This one should be failed by eptc"), yes. tc15(_) -> - exit(byebye). + exit(this_error_must_show). +tc16(_) -> + exit(this_error_must_show). diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_3_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_3_SUITE.erl new file mode 100644 index 0000000000..8271b23afe --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_3_SUITE.erl @@ -0,0 +1,146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_3_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 3). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,?TO}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{g1,[parallel],[tc0,tc1,tc2,tc3,tc4,tc5,tc6,tc7]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [{group,g1}]. + +tc0() -> + [{timetrap,2000}]. +tc0(_) -> + ct:comment("TO after 2 sec"), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,500}]. +tc1(_) -> + ct:comment("TO after 1/2 sec"), + ct:sleep({seconds,5}), + ok. + +tc2() -> + [{timetrap,1000}]. +tc2(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,5}), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc4() -> + [{timetrap,2000}]. +tc4(_) -> + ct:comment(io_lib:format("TO after 2 sec", [])), + ct:sleep({seconds,5}), + ok. + +tc5() -> + [{timetrap,2000}]. +tc5(_) -> + ct:comment("No timeout"), + ct:sleep({seconds,1}), + ok. + +tc6() -> + [{timetrap,1000}]. +tc6(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,5}), + ok. + +tc7() -> + [{timetrap,1500}]. +tc7(_) -> + ct:comment("TO after 1 1/2 sec"), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_4_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_4_SUITE.erl new file mode 100644 index 0000000000..d902454f09 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_4_SUITE.erl @@ -0,0 +1,135 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_4_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2,tc3]. + +tc0(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[2000]}}]. +tc1(_) -> + ct:comment("TO after 2 sec"), + ct:sleep({seconds,5}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep(1000), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_5_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_5_SUITE.erl new file mode 100644 index 0000000000..c5d4b5062e --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_5_SUITE.erl @@ -0,0 +1,155 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_5_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap, fun() -> timetrap_utils:timetrap_val({seconds,?TO}) end}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2,tc3,tc4,tc5,tc6,tc7]. + +tc0(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]. +tc1(_) -> + exit(this_should_not_execute). + +tc2() -> + [{timetrap,fun() -> exit(kaboom) end}]. +tc2(_) -> + exit(this_should_not_execute). + +tc3() -> + [{timetrap,{timetrap_utils,timetrap_err_mfa,[]}}]. +tc3(_) -> + exit(this_should_not_execute). + +tc4() -> + [{timetrap,fun() -> timetrap_utils:timetrap_err_fun() end}]. +tc4(_) -> + exit(this_should_not_execute). + +tc5() -> + [{timetrap,{timetrap_utils,timetrap_timeout,[{seconds,40}, + {seconds,1}]}}]. +tc5(_) -> + ct:comment("TO after 40+1 sec"), + ct:sleep({seconds,42}), + ok. + +tc6() -> + [{timetrap,fun() -> ct:sleep(6000), 1000 end}]. +tc6(_) -> + ct:comment("TO after 6+1 sec"), + ct:sleep({seconds,10}). + +tc7(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_6_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_6_SUITE.erl new file mode 100644 index 0000000000..90467ff752 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_6_SUITE.erl @@ -0,0 +1,114 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_6_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap, fun() -> exit(kaboom) end}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0]. + +tc0(_) -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_7_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_7_SUITE.erl new file mode 100644 index 0000000000..b25b7770a7 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_7_SUITE.erl @@ -0,0 +1,137 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_7_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). +-define(HANG, 6). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_timeout,[{seconds,?HANG}, + {seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2,tc3]. + +tc0(_) -> + ct:comment(io_lib:format("TO after ~w+~w sec", [?HANG,?TO])), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[2000]}}]. +tc1(_) -> + ct:comment("TO after 2 sec"), + ct:sleep({seconds,5}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep(1000), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w+~w sec", [?HANG,?TO])), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_utils.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_utils.erl new file mode 100644 index 0000000000..fcde6cd701 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_utils.erl @@ -0,0 +1,43 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(timetrap_utils). + +-export([timetrap_val/1, + timetrap_err_fun/0, + timetrap_err_mfa/0, + timetrap_exit/1, + timetrap_timeout/2]). + +timetrap_val(Val) -> + Val. + +timetrap_err_fun() -> + fun() -> 5000 end. + +timetrap_err_mfa() -> + {?MODULE,timetrap_val,[5000]}. + +timetrap_exit(Reason) -> + exit(Reason). + +timetrap_timeout(Sleep, Val) -> + ct:sleep(Sleep), + Val. + diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index f33be8a9d4..940d791b15 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -173,16 +173,14 @@ test_events(missing_conf) -> {?eh,start_info,{1,1,2}}, {?eh,tc_start,{ct_framework,ct_init_per_group}}, {?eh,tc_done,{ct_framework,ct_init_per_group,ok}}, - {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc1}}, {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, - {?eh,test_stats,{2,0,{0,0}}}, + {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc2}}, {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, - {?eh,test_stats,{3,0,{0,0}}}, + {?eh,test_stats,{2,0,{0,0}}}, {?eh,tc_start,{ct_framework,ct_end_per_group}}, {?eh,tc_done,{ct_framework,ct_end_per_group,ok}}, - {?eh,test_stats,{4,0,{0,0}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl index 47cea190dd..bda7d91161 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl @@ -143,7 +143,7 @@ tc1(_) -> exit(should_have_been_skipped). tc2(_) -> - exit(should_have_been_skipped). + timeout_in_end_per_testcase. tc3(_) -> timer:sleep(5000). diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index e85e8e6ad3..6df02d12b7 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -23,7 +23,6 @@ %%% -module(ct_test_support). --include_lib("test_server/include/test_server.hrl"). -include_lib("common_test/include/ct_event.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index f77629b4d1..4782a32933 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1,3 +1,3 @@ -COMMON_TEST_VSN = 1.5.4 +COMMON_TEST_VSN = 1.5.5 diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 2cb33a9a69..49f97686a0 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -612,6 +612,7 @@ do_run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> print(minor, "Current directory is ~p\n", [Cwd]), print_timestamp(minor,"Started at "), TCCallback = get(test_server_testcase_callback), + LogOpts = get(test_server_logopts), Ref = make_ref(), OldGLeader = group_leader(), %% Set ourself to group leader for the spawned process @@ -621,7 +622,7 @@ do_run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> fun() -> run_test_case_eval(Mod, Func, Args, Name, Ref, RunInit, TimetrapData, - TCCallback) + LogOpts, TCCallback) end), group_leader(OldGLeader, self()), put(test_server_detected_fail, []), @@ -733,15 +734,23 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> print(Detail,Format,Args), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); {comment,NewComment} -> + NewComment1 = test_server_ctrl:to_string(NewComment), + NewComment2 = test_server_sup:framework_call(format_comment, + [NewComment1], + NewComment1), Terminate1 = case Terminate of {true,{Time,Value,Loc,Opts,_OldComment}} -> - {true,{Time,Value,mod_loc(Loc),Opts,NewComment}}; + {true,{Time,Value,mod_loc(Loc),Opts,NewComment2}}; Other -> Other end, - run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate1,NewComment,CurrConf); - {set_curr_conf,NewCurrConf} -> + run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate1,NewComment2,CurrConf); + {read_comment,From} -> + From ! {self(),read_comment,Comment}, + run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); + {set_curr_conf,From,NewCurrConf} -> + From ! {self(),set_curr_conf,ok}, run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,NewCurrConf); {'EXIT',Pid,{Ref,Time,Value,Loc,Opts}} -> RetVal = {Time/1000000,Value,mod_loc(Loc),Opts,Comment}, @@ -753,7 +762,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case mod_loc(Loc) of {FwMod,FwFunc,framework} -> %% timout during framework call - spawn_fw_call(FwMod,FwFunc,Pid, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid, {framework_error,{timetrap,TVal}}, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate, @@ -779,7 +788,8 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> %% group leader process or io will cause deadlock, %% so we spawn a dedicated process for the operation %% and let the group leader go back to handle io. - spawn_fw_call(Mod,Func,Pid,{timetrap_timeout,TVal}, + spawn_fw_call(Mod,Func,CurrConf,Pid, + {timetrap_timeout,TVal}, Loc1,self(),Comment), undefined end, @@ -790,12 +800,13 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case mod_loc(Loc) of {FwMod,FwFunc,framework} -> %% timout during framework call - spawn_fw_call(FwMod,FwFunc,Pid, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid, {framework_error,{timetrap,TVal}}, unknown,self(),Comment); Loc1 -> {Mod,_Func} = get_mf(Loc1), - spawn_fw_call(Mod,InitOrEnd,Pid,{timetrap_timeout,TVal}, + spawn_fw_call(Mod,InitOrEnd,CurrConf,Pid, + {timetrap_timeout,TVal}, Loc1,self(),Comment) end, run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); @@ -804,7 +815,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case mod_loc(AbortLoc) of {FwMod,FwFunc,framework} -> %% abort during framework call - spawn_fw_call(FwMod,FwFunc,Pid, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid, {framework_error,ErrorMsg}, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate, @@ -828,7 +839,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> {EndConfPid,{Mod,Func},Conf}; _ -> {Mod,Func} = get_mf(Loc1), - spawn_fw_call(Mod,Func,Pid,ErrorMsg, + spawn_fw_call(Mod,Func,CurrConf,Pid,ErrorMsg, Loc1,self(),Comment), undefined end, @@ -839,17 +850,18 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> %% result of an exit(TestCase,kill) call, which is the %% only way to abort a testcase process that traps exits %% (see abort_current_testcase) - spawn_fw_call(undefined,undefined,Pid,testcase_aborted_or_killed, + spawn_fw_call(undefined,undefined,CurrConf,Pid, + testcase_aborted_or_killed, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); {fw_error,{FwMod,FwFunc,FwError}} -> - spawn_fw_call(FwMod,FwFunc,Pid,{framework_error,FwError}, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid,{framework_error,FwError}, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); _Other -> %% the testcase has terminated because of Reason (e.g. an exit %% because a linked process failed) - spawn_fw_call(undefined,undefined,Pid,Reason, + spawn_fw_call(undefined,undefined,CurrConf,Pid,Reason, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf) end; @@ -857,7 +869,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case CurrConf of {EndConfPid,{Mod,Func},_Conf} -> {_Mod,_Func,TCPid,TCExitReason,Loc} = Data, - spawn_fw_call(Mod,Func,TCPid,TCExitReason,Loc,self(),Comment), + spawn_fw_call(Mod,Func,CurrConf,TCPid,TCExitReason,Loc,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,undefined); _ -> run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf) @@ -928,7 +940,7 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) -> ok end, Supervisor ! {self(),end_conf} - end, + end, Pid = spawn_link(EndConfApply), receive {Pid,end_conf} -> @@ -941,50 +953,72 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) -> end, spawn_link(EndConfProc). -spawn_fw_call(Mod,{init_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why, +spawn_fw_call(Mod,{init_per_testcase,Func},_,Pid,{timetrap_timeout,TVal}=Why, Loc,SendTo,Comment) -> FwCall = fun() -> - Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, - %% if init_per_testcase fails, the test case - %% should be skipped - case catch do_end_tc_call(Mod,Func,{Pid,Skip,[[]]},Why) of - {'EXIT',FwEndTCErr} -> - exit({fw_notify_done,end_tc,FwEndTCErr}); - _ -> - ok - end, - %% finished, report back - SendTo ! {self(),fw_notify_done, - {TVal/1000,Skip,Loc,[],Comment}} + Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, + %% if init_per_testcase fails, the test case + %% should be skipped + case catch do_end_tc_call(Mod,Func, Loc, {Pid,Skip,[[]]}, Why) of + {'EXIT',FwEndTCErr} -> + exit({fw_notify_done,end_tc,FwEndTCErr}); + _ -> + ok + end, + %% finished, report back + SendTo ! {self(),fw_notify_done, + {TVal/1000,Skip,Loc,[],Comment}} end, spawn_link(FwCall); -spawn_fw_call(Mod,{end_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why, - Loc,SendTo,_Comment) -> +spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid, + {timetrap_timeout,TVal}=Why,_Loc,SendTo,Comment) -> + %%! This is a temporary fix that keeps Test Server alive during + %%! execution of a parallel test case group, when sometimes + %%! this clause gets called with EndConf == undefined. See OTP-9594 + %%! for more info. + EndConf1 = if EndConf == undefined -> + [{tc_status,{failed,{Mod,end_per_testcase,Why}}}]; + true -> + EndConf + end, FwCall = fun() -> - Conf = [{tc_status,ok}], - %% if end_per_testcase fails, the test case should be - %% reported successful with a warning printed as comment - case catch do_end_tc_call(Mod,Func,{Pid, - {failed,{Mod,end_per_testcase,Why}}, - [Conf]}, Why) of - {'EXIT',FwEndTCErr} -> - exit({fw_notify_done,end_tc,FwEndTCErr}); - _ -> - ok - end, - %% finished, report back - SendTo ! {self(),fw_notify_done, - {TVal/1000,{error,{Mod,end_per_testcase,Why}},Loc,[], - ["<font color=\"red\">" - "WARNING: end_per_testcase timed out!" - "</font>"]}} + {RetVal,Report} = + case proplists:get_value(tc_status, EndConf1) of + undefined -> + E = {failed,{Mod,end_per_testcase,Why}}, + {E,E}; + E = {failed,Reason} -> + {E,{error,Reason}}; + Result -> + E = {failed,{Mod,end_per_testcase,Why}}, + {Result,E} + end, + FailLoc = proplists:get_value(tc_fail_loc, EndConf1), + case catch do_end_tc_call(Mod,Func, FailLoc, + {Pid,Report,[EndConf1]}, Why) of + {'EXIT',FwEndTCErr} -> + exit({fw_notify_done,end_tc,FwEndTCErr}); + _ -> + ok + end, + %% if end_per_testcase fails a warning should be + %% printed as comment + Comment1 = if Comment == "" -> ""; + true -> Comment ++ "<br>" + end, + %% finished, report back + SendTo ! {self(),fw_notify_done, + {TVal/1000,RetVal,FailLoc,[], + [Comment1,"<font color=\"red\">" + "WARNING: end_per_testcase timed out!" + "</font>"]}} end, spawn_link(FwCall); -spawn_fw_call(FwMod,FwFunc,_Pid,{framework_error,FwError},_,SendTo,_Comment) -> +spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo,_Comment) -> FwCall = fun() -> test_server_sup:framework_call(report, [framework_error, @@ -999,7 +1033,7 @@ spawn_fw_call(FwMod,FwFunc,_Pid,{framework_error,FwError},_,SendTo,_Comment) -> end, spawn_link(FwCall); -spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) -> +spawn_fw_call(Mod,Func,_,Pid,Error,Loc,SendTo,Comment) -> FwCall = fun() -> case catch fw_error_notify(Mod,Func,[], @@ -1011,7 +1045,8 @@ spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) -> ok end, Conf = [{tc_status,{failed,timetrap_timeout}}], - case catch do_end_tc_call(Mod,Func,{Pid,Error,[Conf]},Error) of + case catch do_end_tc_call(Mod,Func, Loc, + {Pid,Error,[Conf]},Error) of {'EXIT',FwEndTCErr} -> exit({fw_notify_done,end_tc,FwEndTCErr}); _ -> @@ -1072,8 +1107,9 @@ job_proxy_msgloop() -> %% or sends a message {failed, File, Line} to it's group_leader run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, - TimetrapData, TCCallback) -> - put(test_server_multiply_timetraps,TimetrapData), + TimetrapData, LogOpts, TCCallback) -> + put(test_server_multiply_timetraps, TimetrapData), + put(test_server_logopts, LogOpts), {{Time,Value},Loc,Opts} = case test_server_sup:framework_call(init_tc,[?pl2a(Mod),Func,Args0], @@ -1081,22 +1117,26 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, {ok,Args} -> run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback); Error = {error,_Reason} -> - NewResult = do_end_tc_call(Mod,Func,{Error,Args0}, + Where = {Mod,Func}, + NewResult = do_end_tc_call(Mod,Func, Where, {Error,Args0}, {skip,{failed,Error}}), - {{0,NewResult},{Mod,Func},[]}; + {{0,NewResult},Where,[]}; {fail,Reason} -> Conf = [{tc_status,{failed,Reason}} | hd(Args0)], + Where = {Mod,Func}, fw_error_notify(Mod, Func, Conf, Reason), - NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf]}, + NewResult = do_end_tc_call(Mod,Func, Where, {{error,Reason},[Conf]}, {fail,Reason}), - {{0,NewResult},{Mod,Func},[]}; + {{0,NewResult},Where,[]}; Skip = {skip,_Reason} -> - NewResult = do_end_tc_call(Mod,Func,{Skip,Args0},Skip), - {{0,NewResult},{Mod,Func},[]}; + Where = {Mod,Func}, + NewResult = do_end_tc_call(Mod,Func, Where, {Skip,Args0}, Skip), + {{0,NewResult},Where,[]}; {auto_skip,Reason} -> - NewResult = do_end_tc_call(Mod, Func, {{skip,Reason},Args0}, + Where = {Mod,Func}, + NewResult = do_end_tc_call(Mod,Func, Where, {{skip,Reason},Args0}, {skip,{fw_auto_skip,Reason}}), - {{0,NewResult},{Mod,Func},[]} + {{0,NewResult},Where,[]} end, exit({Ref,Time,Value,Loc,Opts}). @@ -1110,18 +1150,19 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> Skip = {skip,Reason} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}}], - NewRes = do_end_tc_call(Mod,Func,{Skip,[Conf]}, Skip), + NewRes = do_end_tc_call(Mod,Func, Line, {Skip,[Conf]}, Skip), {{0,NewRes},Line,[]}; {skip_and_save,Reason,SaveCfg} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}], - NewRes = do_end_tc_call(Mod, Func, {{skip,Reason},[Conf]}, - {skip, Reason}), + NewRes = do_end_tc_call(Mod,Func, Line, {{skip,Reason},[Conf]}, + {skip,Reason}), {{0,NewRes},Line,[]}; FailTC = {fail,Reason} -> % user fails the testcase EndConf = [{tc_status,{failed,Reason}} | hd(Args)], fw_error_notify(Mod, Func, EndConf, Reason), - NewRes = do_end_tc_call(Mod, Func, {{error,Reason},[EndConf]}, + NewRes = do_end_tc_call(Mod,Func, {Mod,Func}, + {{error,Reason},[EndConf]}, FailTC), {{0,NewRes},{Mod,Func},[]}; {ok,NewConf} -> @@ -1129,47 +1170,61 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> %% call user callback function if defined NewConf1 = user_callback(TCCallback, Mod, Func, init, NewConf), %% save current state in controller loop - group_leader() ! {set_curr_conf,{{Mod,Func},NewConf1}}, + sync_send(group_leader(),set_curr_conf,{{Mod,Func},NewConf1}, + 5000, fun() -> exit(no_answer_from_group_leader) end), put(test_server_loc, {Mod,Func}), %% execute the test case {{T,Return},Loc} = {ts_tc(Mod, Func, [NewConf1]),get_loc()}, {EndConf,TSReturn,FWReturn} = case Return of {E,TCError} when E=='EXIT' ; E==failed -> + ModLoc = mod_loc(Loc), fw_error_notify(Mod, Func, NewConf1, - TCError, mod_loc(Loc)), - {[{tc_status,{failed,TCError}}|NewConf1], + TCError, ModLoc), + {[{tc_status,{failed,TCError}}, + {tc_fail_loc,ModLoc}|NewConf1], Return,{error,TCError}}; SaveCfg={save_config,_} -> {[{tc_status,ok},SaveCfg|NewConf1],Return,ok}; {skip_and_save,Why,SaveCfg} -> Skip = {skip,Why}, - {[{tc_status,{skipped,Why}},{save_config,SaveCfg}|NewConf1], + {[{tc_status,{skipped,Why}}, + {save_config,SaveCfg}|NewConf1], Skip,Skip}; {skip,Why} -> {[{tc_status,{skipped,Why}}|NewConf1],Return,Return}; _ -> {[{tc_status,ok}|NewConf1],Return,ok} end, - %% clear current state in controller loop - group_leader() ! {set_curr_conf,undefined}, %% call user callback function if defined EndConf1 = user_callback(TCCallback, Mod, Func, 'end', EndConf), + %% update current state in controller loop + sync_send(group_leader(),set_curr_conf,EndConf1, + 5000, fun() -> exit(no_answer_from_group_leader) end), {FWReturn1,TSReturn1,EndConf2} = case end_per_testcase(Mod, Func, EndConf1) of SaveCfg1={save_config,_} -> {FWReturn,TSReturn,[SaveCfg1|lists:keydelete(save_config,1, EndConf1)]}; - {fail,ReasonToFail} -> % user has failed the testcase + {fail,ReasonToFail} -> + %% user has failed the testcase fw_error_notify(Mod, Func, EndConf1, ReasonToFail), {{error,ReasonToFail},{failed,ReasonToFail},EndConf1}; - {failed,{_,end_per_testcase,_}} = Failure -> % unexpected termination + {failed,{_,end_per_testcase,_}} = Failure when FWReturn == ok -> + %% unexpected termination in end_per_testcase + %% report this as the result to the framework {Failure,TSReturn,EndConf1}; _ -> + %% test case result should be reported to framework + %% no matter the status of end_per_testcase {FWReturn,TSReturn,EndConf1} end, + %% clear current state in controller loop + sync_send(group_leader(),set_curr_conf,undefined, + 5000, fun() -> exit(no_answer_from_group_leader) end), put(test_server_init_or_end_conf,undefined), - case do_end_tc_call(Mod, Func, {FWReturn1,[EndConf2]}, TSReturn1) of + case do_end_tc_call(Mod,Func, Loc, + {FWReturn1,[EndConf2]}, TSReturn1) of {failed,Reason} = NewReturn -> fw_error_notify(Mod,Func,EndConf2, Reason), {{T,NewReturn},{Mod,Func},[]}; @@ -1193,18 +1248,43 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> %% call user callback function if defined Return1 = user_callback(TCCallback, Mod, Func, 'end', Return), {Return2,Opts} = process_return_val([Return1], Mod, Func, - Args1, Loc, Return1), + Args1, {Mod,Func}, Return1), {{T,Return2},Loc,Opts} end. -do_end_tc_call(M,F,Res,Return) -> +do_end_tc_call(M,F, Loc, Res, Return) -> + FwMod = os:getenv("TEST_SERVER_FRAMEWORK"), + {Mod,Func} = + if FwMod == M ; FwMod == "undefined"; FwMod == false -> + {M,F}; + is_list(Loc) and (length(Loc)>1) -> + %% If failure in other module (M) than suite, try locate + %% suite name in Loc list and call end_tc with Suite:TestCase + %% instead of M:F. + GetSuite = fun(S,TC) -> + case lists:reverse(atom_to_list(S)) of + [$E,$T,$I,$U,$S,$_|_] -> [{S,TC}]; + _ -> [] + end + end, + case lists:flatmap(fun({S,TC,_}) -> GetSuite(S,TC); + ({{S,TC},_}) -> GetSuite(S,TC); + ({S,TC}) -> GetSuite(S,TC); + (_) -> [] + end, Loc) of + [] -> + {M,F}; + [FoundSuite|_] -> + FoundSuite + end; + true -> + {M,F} + end, + Ref = make_ref(), - case os:getenv("TEST_SERVER_FRAMEWORK") of - FW when FW == "ct_framework"; - FW == "undefined"; - FW == false -> + if FwMod == "ct_framework" ; FwMod == "undefined"; FwMod == false -> case test_server_sup:framework_call( - end_tc, [?pl2a(M),F,Res, Return], ok) of + end_tc, [?pl2a(Mod),Func,Res, Return], ok) of {fail,FWReason} -> {failed,FWReason}; ok -> @@ -1217,9 +1297,9 @@ do_end_tc_call(M,F,Res,Return) -> NewReturn -> NewReturn end; - Other -> - case test_server_sup:framework_call( - end_tc, [Other,F,Res], Ref) of + true -> + case test_server_sup:framework_call(FwMod, end_tc, + [?pl2a(Mod),Func,Res], Ref) of {fail,FWReason} -> {failed,FWReason}; _Else -> @@ -1242,7 +1322,7 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) -> true -> % must be return value from end conf case process_return_val1(Return, M,F,A, Loc, Final, []); false -> % must be Config value from init conf case - case do_end_tc_call(M,F,{ok,A}, Return) of + case do_end_tc_call(M, F, Loc, {ok,A}, Return) of {failed, FWReason} = Failed -> fw_error_notify(M,F,A, FWReason), {Failed, []}; @@ -1259,8 +1339,9 @@ process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts) when E=='EXIT'; E==failed -> fw_error_notify(M,F,A, TCError, mod_loc(Loc)), - case do_end_tc_call(M,F,{{error,TCError}, - [[{tc_status,{failed,TCError}}|Args]]}, Failed) of + case do_end_tc_call(M,F, Loc, {{error,TCError}, + [[{tc_status,{failed,TCError}}|Args]]}, + Failed) of {failed,FWReason} -> {{failed,FWReason},SaveOpts}; NewReturn -> @@ -1277,8 +1358,8 @@ process_return_val1([RetVal={Tag,_}|Opts], M,F,A, Loc, _, SaveOpts) when Tag==sk process_return_val1(Opts, M,F,A, Loc, RetVal, SaveOpts); process_return_val1([_|Opts], M,F,A, Loc, Final, SaveOpts) -> process_return_val1(Opts, M,F,A, Loc, Final, SaveOpts); -process_return_val1([], M,F,A, _Loc, Final, SaveOpts) -> - case do_end_tc_call(M,F,{Final,A}, Final) of +process_return_val1([], M,F,A, Loc, Final, SaveOpts) -> + case do_end_tc_call(M,F, Loc, {Final,A}, Final) of {failed,FWReason} -> {{failed,FWReason},SaveOpts}; NewReturn -> @@ -1389,10 +1470,14 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> ok catch throw:Other -> + Comment0 = case read_comment() of + "" -> ""; + Cmt -> Cmt ++ "<br>" + end, set_loc(erlang:get_stacktrace()), - comment(io_lib:format("<font color=\"red\">" + comment(io_lib:format("~s<font color=\"red\">" "WARNING: ~w thrown!" - "</font>\n",[EndFunc])), + "</font>\n",[Comment0,EndFunc])), group_leader() ! {printout,12, "WARNING: ~w thrown!\n" "Reason: ~p\n" @@ -1408,9 +1493,13 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> exit -> {'EXIT',Reason}; error -> {'EXIT',{Reason,Stk}} end, - comment(io_lib:format("<font color=\"red\">" + Comment0 = case read_comment() of + "" -> ""; + Cmt -> Cmt ++ "<br>" + end, + comment(io_lib:format("~s<font color=\"red\">" "WARNING: ~w crashed!" - "</font>\n",[EndFunc])), + "</font>\n",[Comment0,EndFunc])), group_leader() ! {printout,12, "WARNING: ~w crashed!\n" "Reason: ~p\n" @@ -1900,11 +1989,54 @@ time_ms({seconds,N}) -> seconds(N); time_ms({Other,_N}) -> format("=== ERROR: Invalid time specification: ~p. " "Should be seconds, minutes, or hours.~n", [Other]), - exit({invalid_time_spec,Other}); + exit({invalid_time_format,Other}); time_ms(Ms) when is_integer(Ms) -> Ms; time_ms(infinity) -> infinity; -time_ms(Other) -> exit({invalid_time_spec,Other}). +time_ms(Fun) when is_function(Fun) -> + time_ms_apply(Fun); +time_ms({M,F,A}=MFA) when is_atom(M), is_atom(F), is_list(A) -> + time_ms_apply(MFA); +time_ms(Other) -> exit({invalid_time_format,Other}). + +time_ms_apply(Func) -> + time_ms_apply(Func, [5000,30000,60000,infinity]). + +time_ms_apply(Func, TOs) -> + Apply = fun() -> + case Func of + {M,F,A} -> + exit({self(),apply(M, F, A)}); + Fun -> + exit({self(),Fun()}) + end + end, + Pid = spawn(Apply), + Ref = monitor(process, Pid), + time_ms_wait(Func, Pid, Ref, TOs). +time_ms_wait(Func, Pid, Ref, [TO|TOs]) -> + receive + {'DOWN',Ref,process,Pid,{Pid,Result}} -> + time_ms_check(Result); + {'DOWN',Ref,process,Pid,Error} -> + exit({timetrap_error,Error}) + after + TO -> + format("=== WARNING: No return from timetrap function ~p~n", [Func]), + time_ms_wait(Func, Pid, Ref, TOs) + end; +%% this clause will never execute if 'infinity' is in TOs list, that's ok! +time_ms_wait(Func, Pid, Ref, []) -> + demonitor(Ref), + exit(Pid, kill), + exit({timetrap_error,{no_return_from_timetrap_function,Func}}). + +time_ms_check(MFA = {M,F,A}) when is_atom(M), is_atom(F), is_list(A) -> + exit({invalid_time_format,MFA}); +time_ms_check(Fun) when is_function(Fun) -> + exit({invalid_time_format,Fun}); +time_ms_check(Other) -> + time_ms(Other). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timetrap_cancel(Handle) -> ok @@ -1952,6 +2084,19 @@ hours(N) -> trunc(N * 1000 * 60 * 60). minutes(N) -> trunc(N * 1000 * 60). seconds(N) -> trunc(N * 1000). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% sync_send(Pid,Tag,Msg,Timeout,DoAfter) -> Result +%% +sync_send(Pid,Tag,Msg,Timeout,DoAfter) -> + Pid ! {Tag,self(),Msg}, + receive + {Pid,Tag,Result} -> + Result + after Timeout -> + DoAfter() + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timecall(M,F,A) -> {Time,Val} %% Time = float() @@ -2338,6 +2483,21 @@ comment(String) -> group_leader() ! {comment,String}, ok. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% read_comment() -> string() +%% +%% Read the current comment string stored in +%% state during test case execution. +read_comment() -> + MsgLooper = group_leader(), + MsgLooper ! {read_comment,self()}, + receive + {MsgLooper,read_comment,Comment} -> + Comment + after + 5000 -> + "" + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% os_type() -> OsType diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index f3445b742b..4fad86d16d 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -173,7 +173,7 @@ %%% TEST_SERVER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([output/2, print/2, print/3, print_timestamp/2]). -export([start_node/3, stop_node/1, wait_for_node/1, is_release_available/1]). --export([format/1, format/2, format/3]). +-export([format/1, format/2, format/3, to_string/1]). -export([get_target_info/0]). -export([get_hosts/0]). -export([get_target_os_type/0]). @@ -1297,6 +1297,7 @@ terminate(_Reason, State) -> end, kill_all_jobs(State#state.jobs), test_server_node:stop(State#state.target_info), + test_server_h:restore(), ok. kill_all_jobs([{_Name,JobPid}|Jobs]) -> @@ -1349,6 +1350,10 @@ init_tester(Mod, Func, Args, Dir, Name, {SumLev,MajLev,MinLev}, put(test_server_minor_level, MinLev), put(test_server_random_seed, proplists:get_value(random_seed, ExtraTools)), put(test_server_testcase_callback, TCCallback), + %% before first print, read and set logging options + LogOpts = test_server_sup:framework_call(get_logopts, [], []), + put(test_server_logopts, LogOpts), + put(test_server_log_nl, not lists:member(no_nl, LogOpts)), StartedExtraTools = start_extra_tools(ExtraTools), {TimeMy,Result} = ts_tc(Mod, Func, Args), put(test_server_common_io_handler, undefined), @@ -1664,6 +1669,11 @@ do_test_cases(TopCases, SkipCases, Config, TimetrapData) when is_list(TopCases), is_tuple(TimetrapData) -> start_log_file(), + FwMod = + case os:getenv("TEST_SERVER_FRAMEWORK") of + FW when FW =:= false; FW =:= "undefined" -> ?MODULE; + FW -> list_to_atom(FW) + end, case collect_all_cases(TopCases, SkipCases) of {error,Why} -> print(1, "Error starting: ~p", [Why]), @@ -1676,11 +1686,11 @@ do_test_cases(TopCases, SkipCases, put(test_server_cases, N), put(test_server_case_num, 0), TestSpec = - add_init_and_end_per_suite(TestSpec0, undefined, undefined), - + add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod), TI = get_target_info(), - print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]}, - {" (with repeated test cases)",[]})]), + print(1, "Starting test~s", + [print_if_known(N, {", ~w test cases",[N]}, + {" (with repeated test cases)",[]})]), Test = get(test_server_name), test_server_sup:framework_call(report, [tests_start,{Test,N}]), @@ -1709,13 +1719,12 @@ do_test_cases(TopCases, SkipCases, print(html, "<br>Used Erlang ~s in <tt>~s</tt>.\n", [erlang:system_info(version), code:root_dir()]), - case os:getenv("TEST_SERVER_FRAMEWORK") of - FW when FW =:= false; FW =:= "undefined" -> + if FwMod == ?MODULE -> print(html, "<p>Target:<br>\n"), print_who(TI#target_info.host, TI#target_info.username), print(html, "<br>Used Erlang ~s in <tt>~s</tt>.\n", [TI#target_info.version, TI#target_info.root_dir]); - _ -> + true -> case test_server_sup:framework_call(target_info, []) of TargetInfo when is_list(TargetInfo), length(TargetInfo) > 0 -> @@ -1884,11 +1893,12 @@ start_minor_log_file1(Mod, Func, LogDir, AbsName) -> []), SrcListing = downcase(cast_to_list(Mod)) ++ ?src_listing_ext, - case filelib:is_file(filename:join(LogDir, SrcListing)) of - true -> + case {filelib:is_file(filename:join(LogDir, SrcListing)), + lists:member(no_src, get(test_server_logopts))} of + {true,false} -> print(Lev, "<a href=\"~s#~s\">source code for ~p:~p/1</a>\n", [SrcListing,Func,Mod,Func]); - false -> ok + _ -> ok end, io:fwrite(Fd, "<pre>\n", []), @@ -2005,54 +2015,69 @@ copy_html_file(Src, DestDir) -> end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% add_init_and_end_per_suite(TestSpec, Mod, Ref) -> NewTestSpec +%% add_init_and_end_per_suite(TestSpec, Mod, Ref, FwMod) -> NewTestSpec %% %% Expands TestSpec with an initial init_per_suite, and a final %% end_per_suite element, per each discovered suite in the list. -add_init_and_end_per_suite([{make,_,_}=Case|Cases], LastMod, LastRef) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([{skip_case,{{Mod,all},_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> +add_init_and_end_per_suite([{make,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{{Mod,all},_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_end_per_suite_and_skip(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, + LastRef, FwMod) -> + %% if Mod == FwMod, this conf test is (probably) a test case group where + %% the init- and end-functions are missing in the suite, and if so, + %% the suite name should be stored as {suite,Suite} in Props + case proplists:get_value(suite, Props) of + Suite when Suite =/= undefined, Suite =/= LastMod -> + {PreCases, NextMod, NextRef} = + do_add_init_and_end_per_suite(LastMod, LastRef, Suite), + Case1 = {conf,Ref,proplists:delete(suite,Props),{FwMod,Func}}, + PreCases ++ [Case1|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; + _ -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)] + end; +add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) + when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{Mod,_,_}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{Mod,_,_}=Case|Cases], LastMod, LastRef, FwMod) + when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([Case|Cases], LastMod, LastRef)-> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([], _LastMod, undefined) -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([Case|Cases], LastMod, LastRef, FwMod)-> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([], _LastMod, undefined, _FwMod) -> []; -add_init_and_end_per_suite([], _LastMod, skipped_suite) -> +add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) -> []; -add_init_and_end_per_suite([], LastMod, LastRef) -> +add_init_and_end_per_suite([], LastMod, LastRef, _FwMod) -> [{conf,LastRef,[],{LastMod,end_per_suite}}]. do_add_init_and_end_per_suite(LastMod, LastRef, Mod) -> @@ -2101,7 +2126,12 @@ run_test_cases(TestSpec, Config, TimetrapData) -> maybe_open_job_sock(), - html_convert_modules(TestSpec, Config), + case lists:member(no_src, get(test_server_logopts)) of + true -> + ok; + false -> + html_convert_modules(TestSpec, Config) + end, run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []), @@ -2310,7 +2340,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], handle_test_case_io_and_status(), set_io_buffering(undefined), {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]), + test_server_sup:framework_call(report, [tc_auto_skip, + {?pl2a(Mod),Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); _ -> @@ -2318,7 +2349,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% parallel group (io buffering is active) wait_for_cases(Ref), {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]), + test_server_sup:framework_call(report, [tc_auto_skip, + {?pl2a(Mod),Func,Comment}]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -3959,8 +3991,11 @@ progress(ok, _CaseNum, Mod, Func, _Loc, RetVal, Time, case RetVal of {comment,RetComment} -> String = to_string(RetComment), + HtmlCmt = test_server_sup:framework_call(format_comment, + [String], + String), print(major, "=result ok: ~s", [String]), - "<td>" ++ String ++ "</td>"; + "<td>" ++ HtmlCmt ++ "</td>"; _ -> print(major, "=result ok", []), case Comment0 of @@ -4345,14 +4380,18 @@ output_to_fd(Fd, [$=|Msg], internal) -> io:put_chars(Fd, [$=]), io:put_chars(Fd, Msg), io:put_chars(Fd, "\n"); + output_to_fd(Fd, Msg, internal) -> io:put_chars(Fd, [$=,$=,$=,$ ]), io:put_chars(Fd, Msg), io:put_chars(Fd, "\n"); + output_to_fd(Fd, Msg, _Sender) -> io:put_chars(Fd, Msg), - io:put_chars(Fd, "\n"). - + case get(test_server_log_nl) of + false -> ok; + _ -> io:put_chars(Fd, "\n") + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timestamp_filename_get(Leader) -> string() @@ -4665,7 +4704,7 @@ collect_case_invoke(Mod, Case, MFA, St) -> collect_subcases(Mod, Case, MFA, St, Suite) end; _ -> - Suite = test_server_sup:framework_call(get_suite, [?pl2a(Mod),Case],[]), + Suite = test_server_sup:framework_call(get_suite, [?pl2a(Mod),Case], []), collect_subcases(Mod, Case, MFA, St, Suite) end. diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl index ec9be52bd3..875f45eea6 100644 --- a/lib/test_server/src/test_server_sup.erl +++ b/lib/test_server/src/test_server_sup.erl @@ -26,7 +26,7 @@ cleanup_crash_dumps/0, crash_dump_dir/0, tar_crash_dumps/0, get_username/0, get_os_family/0, hostatom/0, hostatom/1, hoststr/0, hoststr/1, - framework_call/2,framework_call/3, + framework_call/2,framework_call/3,framework_call/4, format_loc/1, package_str/1, package_atom/1, call_trace/1]). -include("test_server_internal.hrl"). @@ -541,8 +541,9 @@ format_loc({Mod,Func}) when is_atom(Func) -> format_loc({Mod,Line}) when is_integer(Line) -> %% ?line macro is used ModStr = package_str(Mod), - case lists:reverse(ModStr) of - [$E,$T,$I,$U,$S,$_|_] -> + case {lists:member(no_src, get(test_server_logopts)), + lists:reverse(ModStr)} of + {false,[$E,$T,$I,$U,$S,$_|_]} -> io_lib:format("{~s,<a href=\"~s~s#~w\">~w</a>}", [ModStr,downcase(ModStr),?src_listing_ext, round_to_10(Line),Line]); @@ -558,8 +559,9 @@ format_loc1([{Mod,Func,Line}|Rest]) -> [" ",format_loc1({Mod,Func,Line}),",\n"|format_loc1(Rest)]; format_loc1({Mod,Func,Line}) -> ModStr = package_str(Mod), - case lists:reverse(ModStr) of - [$E,$T,$I,$U,$S,$_|_] -> + case {lists:member(no_src, get(test_server_logopts)), + lists:reverse(ModStr)} of + {false,[$E,$T,$I,$U,$S,$_|_]} -> io_lib:format("{~s,~w,<a href=\"~s~s#~w\">~w</a>}", [ModStr,Func,downcase(ModStr),?src_listing_ext, round_to_10(Line),Line]); diff --git a/lib/test_server/src/ts_erl_config.erl b/lib/test_server/src/ts_erl_config.erl index 640c8ddc9f..3b41f90d55 100644 --- a/lib/test_server/src/ts_erl_config.erl +++ b/lib/test_server/src/ts_erl_config.erl @@ -222,7 +222,6 @@ erl_interface(Vars,OsType) -> end, CrossCompile = case OsType of vxworks -> "true"; - ose -> "true"; _ -> "false" end, [{erl_interface_libpath, filename:nativename(LibPath)}, @@ -329,8 +328,6 @@ sock_libraries({win32, _}) -> sock_libraries({unix, _}) -> ""; % Included in general libraries if needed. sock_libraries(vxworks) -> - ""; -sock_libraries(ose) -> "". link_library(LibName,{win32, _}) -> @@ -339,8 +336,6 @@ link_library(LibName,{unix, _}) -> "lib" ++ LibName ++ ".a"; link_library(LibName,vxworks) -> "lib" ++ LibName ++ ".a"; -link_library(_LibName,ose) -> - ""; link_library(_LibName,_Other) -> exit({link_library, not_supported}). diff --git a/lib/test_server/test/test_server_SUITE.erl b/lib/test_server/test/test_server_SUITE.erl index 4c344717f0..a8532b08ab 100644 --- a/lib/test_server/test/test_server_SUITE.erl +++ b/lib/test_server/test/test_server_SUITE.erl @@ -119,6 +119,11 @@ test_server_conf02_SUITE(Config) -> run_test_server_tests(SuiteName, NCases, NFail, NExpected, NSucc, NUsrSkip, NAutoSkip, NActualSkip, NActualFail, NActualSucc, Config) -> + + ct:log("See test case log files under:~n~p~n", + [filename:join([proplists:get_value(priv_dir, Config), + SuiteName++".logs"])]), + Node = proplists:get_value(node, Config), {ok,_Pid} = rpc:call(Node,test_server_ctrl, start, []), rpc:call(Node, @@ -132,6 +137,7 @@ run_test_server_tests(SuiteName, NCases, NFail, NExpected, NSucc, end), rpc:call(Node,test_server_ctrl, stop, []), + {ok,#suite{ n_cases = NCases, n_cases_failed = NFail, n_cases_expected = NExpected, diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk index 1dd4a84ce9..563c1b6db6 100644 --- a/lib/test_server/vsn.mk +++ b/lib/test_server/vsn.mk @@ -1,2 +1,2 @@ -TEST_SERVER_VSN = 3.4.4 +TEST_SERVER_VSN = 3.4.5 |