diff options
Diffstat (limited to 'lib')
36 files changed, 2096 insertions, 521 deletions
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 57b032b3fd..f58b2ab0a9 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -133,9 +133,15 @@ {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns} | {stylesheet,CSSFile} | {ct_hooks, CTHs}</v> - <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} - | {hours,integer()}</v> + <v> Time = TimeVal | TimeFunc</v> + <v> TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | + {hours,integer()}</v> + <v> TimeFunc = {Mod,Func,Args} | Fun</v> <v> MilliSec = integer()</v> + <v> Mod = atom()</v> + <v> Func = atom()</v> + <v> Args = list()</v> + <v> Fun = fun()</v> <v> Required = Key | {Key,SubKeys}</v> <v> Key = atom()</v> <v> SubKeys = SubKey | [SubKey]</v> @@ -161,7 +167,9 @@ test case is allowed to take (including <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c>). If the timetrap time is exceeded, the test case fails with reason - <c>timetrap_timeout</c>.</p> + <c>timetrap_timeout</c>. If a <c>TimeFunc</c> function is specified, + it will be called initially and must return a value on + <c>TimeVal</c> format.</p> <p>The <c>require</c> tag specifies configuration variables that are required by test cases in the suite. If the required @@ -248,7 +256,7 @@ </type> <desc> - <p> MANDATORY (only if one or more groups are defined) </p> + <p> OPTIONAL </p> <p>This function is called before execution of a test case group. It typically contains initialization which is common for @@ -279,7 +287,7 @@ </type> <desc> - <p> MANDATORY (only if one or more groups are defined) </p> + <p> OPTIONAL </p> <p>This function is called after the execution of a test case group is finished. It is meant to be used for cleaning up after <c>init_per_group/2</c>. @@ -353,9 +361,15 @@ <v> Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns}</v> - <v> Time = MilliSec | {seconds,integer()} | {minutes,integer()} - | {hours,integer()}</v> + <v> Time = TimeVal | TimeFunc</v> + <v> TimeVal = MilliSec | {seconds,integer()} | {minutes,integer()} | + {hours,integer()}</v> + <v> TimeFunc = {Mod,Func,Args} | Fun</v> <v> MilliSec = integer()</v> + <v> Mod = atom()</v> + <v> Func = atom()</v> + <v> Args = list()</v> + <v> Fun = fun()</v> <v> Required = Key | {Key,SubKeys}</v> <v> Key = atom()</v> <v> SubKeys = SubKey | [SubKey]</v> @@ -378,7 +392,9 @@ exceeded, the test case fails with reason <c>timetrap_timeout</c>. <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c> are included in the - timetrap time.</p> + timetrap time. If a <c>TimeFunc</c> function is specified, + it will be called before the test case (or <c>init_per_testcase/2</c>) + and must return a value on <c>TimeVal</c> format.</p> <p>The <c>require</c> tag specifies configuration variables that are required by the test case. If the required diff --git a/lib/common_test/doc/src/config_file_chapter.xml b/lib/common_test/doc/src/config_file_chapter.xml index 59151a73ec..6fc6638bf7 100644 --- a/lib/common_test/doc/src/config_file_chapter.xml +++ b/lib/common_test/doc/src/config_file_chapter.xml @@ -285,7 +285,7 @@ <c>{ok, Config}</c> - if the configuration variables are read successfully, </item> <item> - <c>{error, Error, ErrorDetails}</c> - if the callback module fails to + <c>{error, {Error, ErrorDetails}}</c> - if the callback module fails to proceed with the given configuration parameters. </item> </list> @@ -422,14 +422,14 @@ stop()-> call(Client, Request)-> case whereis(?REGISTERED_NAME) of undefined-> - {error, not_started, Request}; + {error, {not_started, Request}}; Pid-> Pid ! {Client, Request}, receive Reply-> {ok, Reply} after 4000-> - {error, timeout, Request} + {error, {timeout, Request}} end end. diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 1ab563d74f..9045646733 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -83,7 +83,7 @@ <title>Run tests from command line</title> <pre> ct_run [-dir TestDir1 TestDir2 .. TestDirN] | - [-suite Suite1 Suite2 .. SuiteN + [[-dir TestDir] -suite Suite1 Suite2 .. SuiteN [[-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]]] [-step [config | keep_inactive]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] @@ -92,6 +92,7 @@ [-decrypt_key Key] | [-decrypt_file KeyFile] [-label Label] [-logdir LogDir] + [-logopts LogOpts] [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] @@ -117,6 +118,7 @@ [-decrypt_key Key] | [-decrypt_file KeyFile] [-label Label] [-logdir LogDir] + [-logopts LogOpts] [-allow_user_terms] [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] @@ -138,10 +140,11 @@ <pre> ct_run -vts [-browser Browser] [-dir TestDir1 TestDir2 .. TestDirN] | - [-suite Suite [[-group Group] [-case Case]]] + [[dir TestDir] -suite Suite [[-group Group] [-case Case]]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 ConfigString2 and .. and CallbackModuleN ConfigStringN] + [-logopts LogOpts] [-decrypt_key Key] | [-decrypt_file KeyFile] [-include InclDir1 InclDir2 .. InclDirN] [-no_auto_compile] diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 816aa5b1eb..57059f0ba2 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -128,6 +128,15 @@ <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE -case start stop</c></p> <p><c>$ ct_run -suite $SYS1_TEST/setup_SUITE -group installation -case start stop</c></p> + <p>It is also possible to combine the <c>dir</c>, <c>suite</c> and <c>group/case</c> flags. E.g, to run + <c>x_SUITE</c> and <c>y_SUITE</c> in directory <c>testdir</c>:</p> + + <p><c>$ ct_run -dir ./testdir -suite x_SUITE y_SUITE</c></p> + + <p>This has the same effect as calling:</p> + + <p><c>$ ct_run -suite ./testdir/x_SUITE ./testdir/y_SUITE</c></p> + <p>Other flags that may be used with <c>ct_run</c>:</p> <list> <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item> @@ -167,6 +176,8 @@ <item><c><![CDATA[-decrypt_file <key_file>]]></c>, points out a file containing a decryption key for <seealso marker="config_file_chapter#encrypted_config_files">encrypted configuration files</seealso>.</item> <item><c><![CDATA[-basic_html]]></c>, switches off html enhancements that might not be compatible with older browsers.</item> + <item><c><![CDATA[-logopts <opts>]]></c>, makes it possible to modify aspects of the logging behaviour, see + <seealso marker="run_test_chapter#logopts">Log options</seealso> below.</item> </list> <note><p>Directories passed to Common Test may have either relative or absolute paths.</p></note> @@ -324,8 +335,9 @@ are to be executed by Common Test, and those functions only. If the step option <c>config</c> is specified, breakpoints will also be initially set on the configuration functions in the suite, i.e. - <c>init_per_suite/1</c>, <c>end_per_suite/1</c>, <c>init_per_testcase/2</c> - and <c>end_per_testcase/2</c>.</p> + <c>init_per_suite/1</c>, <c>end_per_suite/1</c>, + <c>init_per_group/2</c>, <c>end_per_group/2</c>, + <c>init_per_testcase/2</c> and <c>end_per_testcase/2</c>.</p> <p>Common Test enables the Debugger auto attach feature, which means that for every new interpreted test case function that starts to execute, a new trace window will automatically pop up. (This is because each test @@ -652,6 +664,30 @@ to follow test progress simply by refreshing pages in the HTML browser. Statistics totals are not presented until a test is complete however.</p> + <section> + <marker id="logopts"></marker> + <title>Log options</title> + <p>With the <c>logopts</c> start flag, it's possible to specify + options that modify some aspects of the logging behaviour. + Currently, the following options are available:</p> + <list> + <item><c>no_src</c></item> + <item><c>no_nl</c></item> + </list> + <p>With <c>no_src</c>, the html version of the test suite source + code will not be generated during the test run (and consequently + not be available in the log file system).</p> + <p>With <c>no_nl</c>, Common Test will not add a newline character + (\n) to the end of an output string that it receives from a call to e.g. + <c>io:format/2</c>, and which it prints to the test case log.</p> + <p>For example, if a test is started with:</p> + <p><c>$ ct_run -suite my_SUITE -logopts no_src</c></p> + <p>then printouts during the test made by successive calls to <c>io:format("x")</c>, + will appear in the test case log as:</p> + <p><c>xxx</c></p> + <p>instead of each <c>x</c> printed on a new line, which is the default behaviour.</p> + </section> + </section> <section> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 3f9fdb7121..e35888e68f 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -280,6 +280,8 @@ the timetrap time is exceeded, the test case fails with reason <c>timetrap_timeout</c>. Note that <c>init_per_testcase</c> and <c>end_per_testcase</c> are included in the timetrap time. + Please see the <seealso marker="write_test_chapter#timetraps">Timetrap</seealso> + section for more details. </p> </item> <tag><em><c>userdata</c></em></tag> @@ -699,8 +701,8 @@ </section> <section> - <title>Timetrap timeouts</title> <marker id="timetraps"></marker> + <title>Timetrap timeouts</title> <p>The default time limit for a test case is 30 minutes, unless a <c>timetrap</c> is specified either by the suite info function or a test case info function. The timetrap timeout value defined @@ -723,6 +725,13 @@ multipled by <c>multiply_timetraps</c>, and possibly scaled up if <c>scale_timetraps</c> is enabled, the function <c>ct:sleep/1</c> may be called.</p> + <p>A function (<c>fun</c> or <c>MFA</c>) may be specified as timetrap value + in the suite- and test case info function, e.g:</p> + <p><c>{timetrap,{test_utils,get_timetrap_value,[?MODULE,system_start]}}</c></p> + <p>The function will be called initially by Common Test (before execution + of the suite or the test case) and must return a time value such as an + integer (millisec), or a <c>{SecMinOrHourTag,Time}</c> tuple. More + information can be found in the <c>common_test</c> reference manual.</p> </section> <section> @@ -818,6 +827,3 @@ </list> </section> </chapter> - - - diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 3a96190256..69e15fa246 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -148,10 +148,10 @@ run(TestDirs) -> %%% {auto_compile,Bool} | {multiply_timetraps,M} | {scale_timetraps,Bool} | %%% {repeat,N} | {duration,DurTime} | {until,StopTime} | %%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} | -%%% {refresh_logs,LogDir} | {basic_html,Bool} | +%%% {refresh_logs,LogDir} | {logopts,LogOpts} | {basic_html,Bool} | %%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} %%% TestDirs = [string()] | string() -%%% Suites = [string()] | string() +%%% Suites = [string()] | [atom()] | string() | atom() %%% Cases = [atom()] | atom() %%% Groups = [atom()] | atom() %%% TestSpecs = [string()] | string() @@ -177,6 +177,8 @@ run(TestDirs) -> %%% DecryptKeyOrFile = {key,DecryptKey} | {file,DecryptFile} %%% DecryptKey = string() %%% DecryptFile = string() +%%% LogOpts = [LogOpt] +%%% LogOpt = no_nl | no_src %%% CTHs = [CTHModule | {CTHModule, CTHInitArgs}] %%% CTHModule = atom() %%% CTHInitArgs = term() diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index 6b75937668..fc51aea7f3 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -204,9 +204,9 @@ get_config_file_list(Opts) -> DefaultConfigs = process_default_configs(Opts), CfgFiles = if - DefaultConfigs == []-> + DefaultConfigs == [] -> []; - true-> + true -> [{?ct_config_txt, DefaultConfigs}] end ++ process_user_configs(Opts, []), @@ -240,12 +240,12 @@ read_config_files(Opts) -> end, ConfigFiles = case lists:keyfind(config, 1, Opts) of - {config,ConfigLists}-> + {config,ConfigLists} -> lists:foldr(fun({Callback,Files}, Acc) -> AddCallback(Callback,Files) ++ Acc end,[],ConfigLists); - false-> + false -> [] end, read_config_files_int(ConfigFiles, fun store_config/3). @@ -255,7 +255,9 @@ read_config_files_int([{Callback, File}|Files], FunToSave) -> {ok, Config} -> FunToSave(Config, Callback, File), read_config_files_int(Files, FunToSave); - {error, ErrorName, ErrorDetail}-> + {error, {ErrorName, ErrorDetail}} -> + {user_error, {ErrorName, File, ErrorDetail}}; + {error, ErrorName, ErrorDetail} -> {user_error, {ErrorName, File, ErrorDetail}} end; read_config_files_int([], _FunToSave) -> @@ -283,7 +285,7 @@ rewrite_config(Config, Callback, File) -> config=File,_='_'}), Updater = fun({Key, Value}) -> case keyfindall(Key, #ct_conf.key, OldRows) of - []-> + [] -> ets:insert(?attr_table, #ct_conf{key=Key, value=Value, @@ -453,9 +455,9 @@ update_conf(Name, NewConfig) -> reload_conf(KeyOrName) -> case lookup_handler_for_config(KeyOrName) of - []-> + [] -> undefined; - HandlerList-> + HandlerList -> HandlerList2 = lists:usort(HandlerList), read_config_files_int(HandlerList2, fun rewrite_config/3), get_config(KeyOrName) @@ -711,13 +713,13 @@ random_bytes_1(N, Acc) -> random_bytes_1(N-1, [random:uniform(255)|Acc]). check_callback_load(Callback) -> case code:is_loaded(Callback) of - {file, _Filename}-> + {file, _Filename} -> check_exports(Callback); - false-> + false -> case code:load_file(Callback) of - {module, Callback}-> + {module, Callback} -> check_exports(Callback); - {error, Error}-> + {error, Error} -> {error, Error} end end. @@ -745,14 +747,14 @@ check_config_files(Configs) -> end, Files) end; - {error, Why}-> + {error, Why} -> {error, {callback, {Callback,Why}}} end; ({Callback, []}) -> case check_callback_load(Callback) of - {ok, Callback}-> + {ok, Callback} -> Callback:check_parameter([]); - {error, Why}-> + {error, Why} -> {error, {callback, {Callback,Why}}} end end, @@ -773,15 +775,15 @@ prepare_user_configs([], Acc, _) -> prepare_config_list(Args) -> ConfigFiles = case lists:keysearch(ct_config, 1, Args) of - {value,{ct_config,Files}}-> + {value,{ct_config,Files}} -> [{?ct_config_txt,[filename:absname(F) || F <- Files]}]; - false-> + false -> [] end, UserConfigs = case lists:keysearch(userconfig, 1, Args) of - {value,{userconfig,UserConfigFiles}}-> + {value,{userconfig,UserConfigFiles}} -> prepare_user_configs(UserConfigFiles, [], new); - false-> + false -> [] end, ConfigFiles ++ UserConfigs. diff --git a/lib/common_test/src/ct_config_plain.erl b/lib/common_test/src/ct_config_plain.erl index 3fbc8af9fb..6698332379 100644 --- a/lib/common_test/src/ct_config_plain.erl +++ b/lib/common_test/src/ct_config_plain.erl @@ -29,7 +29,7 @@ read_config(ConfigFile) -> {ok,Config} -> {ok, Config}; {error,enoent} -> - {error, config_file_error, enoent}; + {error,{config_file_error,file:format_error(enoent)}}; {error,Reason} -> Key = case application:get_env(common_test, decrypt) of @@ -45,23 +45,27 @@ read_config(ConfigFile) -> end, case Key of {error,no_crypt_file} -> - {error, config_file_error, Reason}; + {error,{config_file_error, + lists:flatten( + io_lib:format("~s",[file:format_error(Reason)]))}}; {error,CryptError} -> - {error, decrypt_file_error, CryptError}; + {error,{decrypt_file_error,CryptError}}; _ when is_list(Key) -> - case ct_config:decrypt_config_file(ConfigFile, undefined, {key,Key}) of + case ct_config:decrypt_config_file(ConfigFile, + undefined, + {key,Key}) of {ok,CfgBin} -> case read_config_terms(CfgBin) of {error,ReadFail} -> - {error, config_file_error, ReadFail}; + {error,{config_file_error,ReadFail}}; Config -> - {ok, Config} + {ok,Config} end; {error,DecryptFail} -> - {error, decrypt_config_error, DecryptFail} + {error,{decrypt_config_error,DecryptFail}} end; _ -> - {error, bad_decrypt_key, Key} + {error,{bad_decrypt_key,Key}} end end. @@ -69,9 +73,9 @@ read_config(ConfigFile) -> check_parameter(File)-> case filelib:is_file(File) of true-> - {ok, {file, File}}; + {ok,{file,File}}; false-> - {error, {nofile, File}} + {error,{nofile,File}} end. read_config_terms(Bin) when is_binary(Bin) -> diff --git a/lib/common_test/src/ct_config_xml.erl b/lib/common_test/src/ct_config_xml.erl index 8a6e75e635..794174e663 100644 --- a/lib/common_test/src/ct_config_xml.erl +++ b/lib/common_test/src/ct_config_xml.erl @@ -27,30 +27,30 @@ % read config file read_config(ConfigFile) -> case catch do_read_xml_config(ConfigFile) of - {ok, Config}-> - {ok, Config}; - {error, Error, ErroneousString}-> - {error, Error, ErroneousString} + {ok,Config} -> + {ok,Config}; + Error = {error,_} -> + Error end. % check file exists -check_parameter(File)-> +check_parameter(File) -> case filelib:is_file(File) of - true-> - {ok, {file, File}}; - false-> - {error, {nofile, File}} + true -> + {ok,{file,File}}; + false -> + {error,{nofile,File}} end. % actual reading of the config -do_read_xml_config(ConfigFile)-> +do_read_xml_config(ConfigFile) -> case catch xmerl_sax_parser:file(ConfigFile, - [{event_fun, fun event/3}, - {event_state, []}]) of - {ok, EntityList, _}-> - {ok, lists:reverse(transform_entity_list(EntityList))}; - Oops-> - {error, parsing_failed, Oops} + [{event_fun,fun event/3}, + {event_state,[]}]) of + {ok,EntityList,_} -> + {ok,lists:reverse(transform_entity_list(EntityList))}; + Oops -> + {error,{parsing_failed,Oops}} end. % event callback for xmerl_sax_parser @@ -92,18 +92,18 @@ tag(_El, State) -> State. % transform of the ugly deeply nested entity list to the key-value "tree" -transform_entity_list(EntityList)-> +transform_entity_list(EntityList) -> lists:map(fun transform_entity/1, EntityList). % transform entity from {list(), list()} to {atom(), term()} transform_entity({Tag, [Value|Rest]}) when - is_tuple(Value)-> + is_tuple(Value) -> {list_to_atom(Tag), transform_entity_list(lists:reverse([Value|Rest]))}; -transform_entity({Tag, String})-> +transform_entity({Tag, String}) -> case list_to_term(String) of - {ok, Value}-> + {ok, Value} -> {list_to_atom(Tag), Value}; - Error-> + Error -> throw(Error) end. @@ -111,8 +111,8 @@ transform_entity({Tag, String})-> list_to_term(String) -> {ok, T, _} = erl_scan:string(String++"."), case catch erl_parse:parse_term(T) of - {ok, Term} -> - {ok, Term}; + {ok,Term} -> + {ok,Term}; Error -> - {error, Error, String} + {error,{Error,String}} end. diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 2ebc6c311a..482c5242ce 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -24,10 +24,10 @@ -module(ct_framework). --export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, report/2, warn/1]). --export([error_notification/4]). +-export([init_tc/3, end_tc/3, end_tc/4, get_suite/2, get_all_cases/1]). +-export([report/2, warn/1, error_notification/4]). --export([overview_html_header/1]). +-export([get_logopts/0, format_comment/1, overview_html_header/1]). -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). @@ -116,7 +116,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> Config = lists:keydelete(watchdog,1,Config1), if Func /= init_per_suite, DoInit /= true -> ok; - true -> + true -> %% delete all default values used in previous suite ct_config:delete_default_config(suite), %% release all name -> key bindings (once per suite) @@ -133,7 +133,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> ct_config:delete_default_config(testcase), case add_defaults(Mod,Func,TestCaseInfo,DoInit) of Error = {suite0_failed,_} -> - ct_logs:init_tc(), + ct_logs:init_tc(false), FuncSpec = group_or_func(Func,Config0), ct_event:notify(#event{name=tc_start, node=node(), @@ -143,7 +143,7 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} when DoInit == false -> - ct_logs:init_tc(), + ct_logs:init_tc(false), FuncSpec = group_or_func(Func,Config0), ct_event:notify(#event{name=tc_start, node=node(), @@ -194,19 +194,24 @@ init_tc2(Mod,Func,SuiteInfo,MergeResult,Config,DoInit) -> Conns -> ct_util:silence_connections(Conns) end, - - ct_logs:init_tc(), + if Func /= init_per_suite, DoInit /= true -> + ct_logs:init_tc(false); + true -> + ct_logs:init_tc(true) + end, FuncSpec = group_or_func(Func,Config), ct_event:notify(#event{name=tc_start, node=node(), data={Mod,FuncSpec}}), - case configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of + case catch configure(MergedInfo1,MergedInfo1,SuiteInfo,{Func,DoInit},Config) of {suite0_failed,Reason} -> ct_util:set_testdata({curr_tc,{Mod,{suite0_failed,{require,Reason}}}}), {skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; + {'EXIT',Reason} -> + {auto_skip,Reason}; {ok, FinalConfig} -> case MergeResult of {error,Reason} -> @@ -245,7 +250,12 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> Error = {error,_} -> {SuiteInfo,Error}; MergedInfo -> {SuiteInfo,MergedInfo} end; - {'EXIT',Reason} -> + {'EXIT',Reason} -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:suite/0 failed: ~p~n", + [Mod,Reason]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,{exited,Reason}}; SuiteInfo when is_list(SuiteInfo) -> case lists:all(fun(E) when is_tuple(E) -> true; @@ -261,9 +271,19 @@ add_defaults(Mod,Func,FuncInfo,DoInit) -> MergedInfo -> {SuiteInfo1,MergedInfo} end; false -> + ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:suite/0: ~p~n", [Mod,SuiteInfo]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end; - _ -> + SuiteInfo -> + ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:suite/0: ~p~n", [Mod,SuiteInfo]), + io:format(ErrStr, []), + io:format(user, ErrStr, []), {suite0_failed,bad_return_value} end. @@ -451,7 +471,6 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> {value,{watchdog,Dog}} -> test_server:timetrap_cancel(Dog); false -> ok end, - %% save the testcase process pid so that it can be used %% to look up the attached trace window later case ct_util:get_testdata(interpret) of @@ -461,7 +480,6 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> _ -> ok end, - ct_util:delete_testdata(comment), ct_util:delete_suite_data(last_saved_config), FuncSpec = @@ -767,6 +785,37 @@ get_suite(Mod, Name) -> %%%----------------------------------------------------------------- +get_all_cases(Suite) -> + case get_suite(Suite, all) of + [{?MODULE,error_in_suite,[[{error,_}=Error]]}] -> + Error; + [{?MODULE,error_in_suite,[[Error]]}] -> + {error,Error}; + Tests -> + Cases = get_all_cases1(Suite, Tests), + lists:reverse( + lists:foldl(fun(TC, TCs) -> + case lists:member(TC, TCs) of + true -> TCs; + false -> [TC | TCs] + end + end, [], Cases)) + end. + +get_all_cases1(Suite, [{conf,_Props,_Init,GrTests,_End} | Tests]) -> + get_all_cases1(Suite, GrTests) ++ get_all_cases1(Suite, Tests); + +get_all_cases1(Suite, [Test | Tests]) when is_atom(Test) -> + [{Suite,Test} | get_all_cases1(Suite, Tests)]; + +get_all_cases1(Suite, [Test | Tests]) -> + [Test | get_all_cases1(Suite, Tests)]; + +get_all_cases1(_, []) -> + []. + +%%%----------------------------------------------------------------- + find_groups(Mod, Name, TCs, GroupDefs) -> Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), trim(Found). @@ -978,15 +1027,20 @@ make_conf(Mod, Name, Props, TestSpec) -> _ -> ok end, - {InitConf,EndConf} = + {InitConf,EndConf,ExtraProps} = case erlang:function_exported(Mod,init_per_group,2) of true -> - {{Mod,init_per_group},{Mod,end_per_group}}; + {{Mod,init_per_group},{Mod,end_per_group},[]}; false -> + ct_logs:log("TEST INFO", "init_per_group/2 and " + "end_per_group/2 missing for group " + "~p in ~p, using default.", + [Name,Mod]), {{?MODULE,ct_init_per_group}, - {?MODULE,ct_end_per_group}} + {?MODULE,ct_end_per_group}, + [{suite,Mod}]} end, - {conf,[{name,Name}|Props],InitConf,TestSpec,EndConf}. + {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. %%%----------------------------------------------------------------- @@ -1159,13 +1213,15 @@ error_in_suite(Config) -> %% if the group config functions are missing in the suite, %% use these instead ct_init_per_group(GroupName, Config) -> - ct_logs:log("WARNING", "init_per_group/2 for ~w missing " + ct:comment(io_lib:format("start of ~p", [GroupName])), + ct_logs:log("TEST INFO", "init_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), Config. ct_end_per_group(GroupName, _) -> - ct_logs:log("WARNING", "end_per_group/2 for ~w missing " + ct:comment(io_lib:format("end of ~p", [GroupName])), + ct_logs:log("TEST INFO", "end_per_group/2 for ~w missing " "in suite, using default.", [GroupName]), ok. @@ -1242,12 +1298,20 @@ report(What,Data) -> ok; {end_per_group,_} -> ok; + {ct_init_per_group,_} -> + ok; + {ct_end_per_group,_} -> + ok; {_,ok} -> add_to_stats(ok); {_,{skipped,{failed,{_,init_per_testcase,_}}}} -> add_to_stats(auto_skipped); {_,{skipped,{require_failed,_}}} -> add_to_stats(auto_skipped); + {_,{skipped,{timetrap_error,_}}} -> + add_to_stats(auto_skipped); + {_,{skipped,{invalid_time_format,_}}} -> + add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); {_,{SkipOrFail,_Reason}} -> @@ -1332,6 +1396,21 @@ add_data_dir(File,Config) when is_list(File) -> end. %%%----------------------------------------------------------------- +%%% @spec get_logopts() -> [LogOpt] +get_logopts() -> + case ct_util:get_testdata(logopts) of + undefined -> + []; + LogOpts -> + LogOpts + end. + +%%%----------------------------------------------------------------- +%%% @spec format_comment(Comment) -> HtmlComment +format_comment(Comment) -> + "<font color=\"green\">" ++ Comment ++ "</font>". + +%%%----------------------------------------------------------------- %%% @spec overview_html_header(TestName) -> Header overview_html_header(TestName) -> TestName1 = lists:flatten(io_lib:format("~p", [TestName])), diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 6a90441d53..c1523509a5 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -28,7 +28,7 @@ -module(ct_logs). --export([init/1,close/1,init_tc/0,end_tc/1]). +-export([init/1,close/2,init_tc/1,end_tc/1]). -export([get_log_dir/0,log/3,start_log/1,cont_log/2,end_log/0]). -export([set_stylesheet/2,clear_stylesheet/1]). -export([add_external_logs/1,add_link/3]). @@ -97,11 +97,11 @@ logdir_node_prefix() -> logdir_prefix()++"."++atom_to_list(node()). %%%----------------------------------------------------------------- -%%% @spec close(Info) -> ok +%%% @spec close(Info, StartDir) -> ok %%% %%% @doc Create index pages with test results and close the CT Log %%% (tool-internal use only). -close(Info) -> +close(Info, StartDir) -> make_last_run_index(), ct_event:notify(#event{name=stop_logging,node=node(),data=[]}), @@ -124,14 +124,29 @@ close(Info) -> ok; Error -> io:format("Warning! Cleanup failed: ~p~n", [Error]) - end; + end, + make_all_suites_index(stop), + make_all_runs_index(stop); true -> - file:set_cwd("..") - end, - - make_all_suites_index(stop), - make_all_runs_index(stop), - + file:set_cwd(".."), + make_all_suites_index(stop), + make_all_runs_index(stop), + case ct_util:get_profile_data(browser, StartDir) of + undefined -> + ok; + BrowserData -> + case {proplists:get_value(prog, BrowserData), + proplists:get_value(args, BrowserData), + proplists:get_value(page, BrowserData)} of + {Prog,Args,Page} when is_list(Args), + is_list(Page) -> + URL = "\"file://" ++ ?abs(Page) ++ "\"", + ct_util:open_url(Prog, Args, URL); + _ -> + ok + end + end + end, ok. %%%----------------------------------------------------------------- @@ -182,15 +197,14 @@ cast(Msg) -> ?MODULE ! Msg end. - %%%----------------------------------------------------------------- -%%% @spec init_tc() -> ok +%%% @spec init_tc(RefreshLog) -> ok %%% %%% @doc Test case initiation (tool-internal use only). %%% %%% <p>This function is called by ct_framework:init_tc/3</p> -init_tc() -> - call({init_tc,self(),group_leader()}), +init_tc(RefreshLog) -> + call({init_tc,self(),group_leader(),RefreshLog}), ok. %%%----------------------------------------------------------------- @@ -486,8 +500,8 @@ logger_loop(State) -> [Str,Args]), %% stop the testcase, we need %% to see the fault - exit(Pid,logging_failed), - ok; + exit(Pid,{log_printout_error,Str,Args}), + []; IoStr when IoList == [] -> [IoStr]; IoStr -> @@ -507,10 +521,15 @@ logger_loop(State) -> [begin io:format(Fd,Str,Args),io:nl(Fd) end || {Str,Args} <- List], logger_loop(State#logger_state{tc_groupleaders=TCGLs}) end; - {{init_tc,TCPid,GL},From} -> + {{init_tc,TCPid,GL,RefreshLog},From} -> print_style(GL, State#logger_state.stylesheet), set_evmgr_gl(GL), TCGLs = add_tc_gl(TCPid,GL,State), + if not RefreshLog -> + ok; + true -> + make_last_run_index(State#logger_state.start_time) + end, return(From,ok), logger_loop(State#logger_state{tc_groupleaders=TCGLs}); {{end_tc,TCPid},From} -> @@ -841,6 +860,7 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, "" end end, + CtRunDir = filename:dirname(filename:dirname(Link)), {Lbl,Timestamp,Node,AllInfo} = case All of {true,OldRuns} -> @@ -850,7 +870,6 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, _ -> NodeOrDate end, N = ["<TD ALIGN=right><FONT SIZE=-1>",Node1,"</FONT></TD>\n"], - CtRunDir = filename:dirname(filename:dirname(Link)), L = ["<TD ALIGN=center><FONT SIZE=-1><B>",Label,"</FONT></B></TD>\n"], T = ["<TD><FONT SIZE=-1>",timestamp(CtRunDir),"</FONT></TD>\n"], CtLogFile = filename:join(CtRunDir,?ct_log_name), @@ -869,7 +888,7 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, if NotBuilt == 0 -> ["<TD ALIGN=right>",integer_to_list(NotBuilt),"</TD>\n"]; true -> - ["<TD ALIGN=right><A HREF=\"",?ct_log_name,"\">", + ["<TD ALIGN=right><A HREF=\"",filename:join(CtRunDir,?ct_log_name),"\">", integer_to_list(NotBuilt),"</A></TD>\n"] end, FailStr = diff --git a/lib/common_test/src/ct_make.erl b/lib/common_test/src/ct_make.erl index 233e45248e..40e9e99f37 100644 --- a/lib/common_test/src/ct_make.erl +++ b/lib/common_test/src/ct_make.erl @@ -177,7 +177,7 @@ members([],_MakefileMods,I,Rest) -> {I,Rest}. -%% Any flags that are not recognixed as make flags are passed directly +%% Any flags that are not recognised as make flags are passed directly %% to the compiler. %% So for example make:all([load,debug_info]) will make everything %% with the debug_info flag and load it. diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 0715b8abf8..0a9bb5af67 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -33,7 +33,7 @@ %% Exported for VTS --export([run_make/3,do_run/3,tests/1,tests/2,tests/3]). +-export([run_make/3,do_run/4,tests/1,tests/2,tests/3]). %% Misc internal functions @@ -46,12 +46,14 @@ -define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)). -record(opts, {label, + profile, vts, shell, cover, coverspec, step, logdir, + logopts = [], config = [], event_handlers = [], ct_hooks = [], @@ -157,15 +159,19 @@ script_start(Args) -> end, stop_trace(Tracing), timer:sleep(1000), + io:nl(), Res. script_start1(Parent, Args) -> %% read general start flags Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args), + Profile = get_start_opt(profile, fun([Prof]) -> Prof end, Args), Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), + LogOpts = get_start_opt(logopts, fun(Os) -> [list_to_atom(O) || O <- Os] end, + [], Args), MultTT = get_start_opt(multiply_timetraps, fun([MT]) -> list_to_integer(MT) end, 1, Args), ScaleTT = get_start_opt(scale_timetraps, @@ -239,8 +245,10 @@ script_start1(Parent, Args) -> application:set_env(common_test, basic_html, true) end, - StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover, - logdir = LogDir, event_handlers = EvHandlers, + StartOpts = #opts{label = Label, profile = Profile, + vts = Vts, shell = Shell, cover = Cover, + logdir = LogDir, logopts = LogOpts, + event_handlers = EvHandlers, ct_hooks = CTHooks, enable_builtin_hooks = EnableBuiltinHooks, include = IncludeDirs, @@ -303,9 +311,15 @@ script_start2(StartOpts = #opts{vts = undefined, Label = choose_val(StartOpts#opts.label, SpecStartOpts#opts.label), + Profile = choose_val(StartOpts#opts.profile, + SpecStartOpts#opts.profile), + LogDir = choose_val(StartOpts#opts.logdir, SpecStartOpts#opts.logdir), + AllLogOpts = merge_vals([StartOpts#opts.logopts, + SpecStartOpts#opts.logopts]), + Cover = choose_val(StartOpts#opts.cover, SpecStartOpts#opts.cover), MultTT = choose_val(StartOpts#opts.multiply_timetraps, @@ -328,9 +342,11 @@ script_start2(StartOpts = #opts{vts = undefined, application:set_env(common_test, include, AllInclude), {TS,StartOpts#opts{label = Label, + profile = Profile, testspecs = Specs, cover = Cover, logdir = LogDir, + logopts = AllLogOpts, config = SpecStartOpts#opts.config, event_handlers = AllEvHs, ct_hooks = AllCTHooks, @@ -404,50 +420,72 @@ check_and_install_configfiles( end. script_start3(StartOpts, Args) -> - case proplists:get_value(dir, Args) of - [] -> + StartOpts1 = get_start_opt(step, + fun(Step) -> + StartOpts#opts{step = Step, + cover = undefined} + end, StartOpts, Args), + case {proplists:get_value(dir, Args), + proplists:get_value(suite, Args), + groups_and_cases(proplists:get_value(group, Args), + proplists:get_value(testcase, Args))} of + %% flag specified without data + {_,_,Error={error,_}} -> + Error; + {_,[],_} -> + {error,no_suite_specified}; + {[],_,_} -> {error,no_dir_specified}; - Dirs when is_list(Dirs) -> + + {Dirs,undefined,[]} when is_list(Dirs) -> script_start4(StartOpts#opts{tests = tests(Dirs)}, Args); - undefined -> - case proplists:get_value(suite, Args) of - [] -> - {error,no_suite_specified}; - Suites when is_list(Suites) -> - StartOpts1 = - get_start_opt(step, - fun(Step) -> - StartOpts#opts{step = Step, - cover = undefined} - end, StartOpts, Args), - DirMods = [suite_to_test(S) || S <- Suites], - case groups_and_cases(proplists:get_value(group, Args), - proplists:get_value(testcase, Args)) of - Error = {error,_} -> - Error; - [] when DirMods =/= [] -> - Ts = tests(DirMods), - script_start4(StartOpts1#opts{tests = Ts}, Args); - GroupsAndCases when length(DirMods) == 1 -> - Ts = tests(DirMods, GroupsAndCases), - script_start4(StartOpts1#opts{tests = Ts}, Args); - [_,_|_] when length(DirMods) > 1 -> - {error,multiple_suites_and_cases}; - _ -> - {error,incorrect_suite_option} - end; - undefined -> - if StartOpts#opts.vts ; StartOpts#opts.shell -> - script_start4(StartOpts#opts{tests = []}, Args); - true -> - script_usage(), - {error,incorrect_usage} - end + + {undefined,Suites,[]} when is_list(Suites) -> + Ts = tests([suite_to_test(S) || S <- Suites]), + script_start4(StartOpts1#opts{tests = Ts}, Args); + + {undefined,Suite,GsAndCs} when is_list(Suite) -> + case [suite_to_test(S) || S <- Suite] of + DirMods = [_] -> + Ts = tests(DirMods, GsAndCs), + script_start4(StartOpts1#opts{tests = Ts}, Args); + [_,_|_] -> + {error,multiple_suites_and_cases}; + _ -> + {error,incorrect_start_options} + end; + + {[_,_|_],Suite,[]} when is_list(Suite) -> + {error,multiple_dirs_and_suites}; + + {[Dir],Suite,GsAndCs} when is_list(Dir), is_list(Suite) -> + case [suite_to_test(Dir,S) || S <- Suite] of + DirMods when GsAndCs == [] -> + Ts = tests(DirMods), + script_start4(StartOpts1#opts{tests = Ts}, Args); + DirMods = [_] when GsAndCs /= [] -> + Ts = tests(DirMods, GsAndCs), + script_start4(StartOpts1#opts{tests = Ts}, Args); + [_,_|_] when GsAndCs /= [] -> + {error,multiple_suites_and_cases}; + _ -> + {error,incorrect_start_options} + end; + + {undefined,undefined,GsAndCs} when GsAndCs /= [] -> + {error,incorrect_start_options}; + + {undefined,undefined,_} -> + if StartOpts#opts.vts ; StartOpts#opts.shell -> + script_start4(StartOpts#opts{tests = []}, Args); + true -> + script_usage(), + {error,missing_start_options} end end. script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, - tests = Tests, logdir = LogDir}, _Args) -> + tests = Tests, logdir = LogDir, logopts = LogOpts}, _Args) -> ConfigFiles = lists:foldl(fun({ct_config_plain,CfgFiles}, AllFiles) when is_list(hd(CfgFiles)) -> @@ -458,16 +496,21 @@ script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, (_, AllFiles) -> AllFiles end, [], Config), - vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests); + vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), LogOpts, Tests); -script_start4(#opts{label = Label, shell = true, config = Config, +script_start4(#opts{label = Label, profile = Profile, + shell = true, config = Config, event_handlers = EvHandlers, ct_hooks = CTHooks, + logopts = LogOpts, enable_builtin_hooks = EnableBuiltinHooks, logdir = LogDir, testspecs = Specs}, _Args) -> %% label - used by ct_logs application:set_env(common_test, test_label, Label), + %% profile - used in ct_util + application:set_env(common_test, profile, Profile), + if Config == [] -> ok; true -> @@ -478,6 +521,7 @@ script_start4(#opts{label = Label, shell = true, config = Config, {enable_builtin_hooks,EnableBuiltinHooks}]) of ok -> ct_util:start(interactive, LogDir), + ct_util:set_testdata({logopts, LogOpts}), log_ts_names(Specs), io:nl(), ok; @@ -518,6 +562,7 @@ script_usage() -> "\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]" "\n\t[-dir TestDir1 TestDir2 .. TestDirN] |" "\n\t[-suite Suite [-case Case]]" + "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -534,8 +579,9 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" - "\n\t[-event_handler EvHandler1 and EvHandler2 .. EvHandlerN]" - "\n\t[-ct_hooks CTHook1 and CTHook2 .. CTHookN]" + "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -553,8 +599,9 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" - "\n\t[-event_handler EvHandler1 and EvHandler2 .. EvHandlerN]" - "\n\t[-ct_hooks CTHook1 and CTHook2 .. CTHookN]" + "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" + "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" + "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" "\n\t[-no_auto_compile]" "\n\t[-multiply_timetraps N]" @@ -632,6 +679,16 @@ run_test(StartOpt) when is_tuple(StartOpt) -> run_test([StartOpt]); run_test(StartOpts) when is_list(StartOpts) -> + CTPid = spawn(fun() -> run_test1(StartOpts) end), + Ref = monitor(process, CTPid), + receive + {'DOWN',Ref,process,CTPid,{user_error,Error}} -> + Error; + {'DOWN',Ref,process,CTPid,Other} -> + Other + end. + +run_test1(StartOpts) when is_list(StartOpts) -> case proplists:get_value(refresh_logs, StartOpts) of undefined -> Tracing = start_trace(StartOpts), @@ -640,7 +697,7 @@ run_test(StartOpts) when is_list(StartOpts) -> Res = case ct_repeat:loop_test(func, StartOpts) of false -> - case catch run_test1(StartOpts) of + case catch run_test2(StartOpts) of {'EXIT',Reason} -> file:set_cwd(Cwd), {error,Reason}; @@ -651,20 +708,27 @@ run_test(StartOpts) when is_list(StartOpts) -> Result end, stop_trace(Tracing), - Res; + exit(Res); RefreshDir -> refresh_logs(?abs(RefreshDir)), - ok + exit(done) end. -run_test1(StartOpts) -> +run_test2(StartOpts) -> %% label Label = get_start_opt(label, fun(Lbl) when is_list(Lbl) -> Lbl; (Lbl) when is_atom(Lbl) -> atom_to_list(Lbl) end, StartOpts), + %% profile + Profile = get_start_opt(profile, fun(Prof) when is_list(Prof) -> Prof; + (Prof) when is_atom(Prof) -> atom_to_list(Prof) + end, StartOpts), %% logdir LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end, StartOpts), + %% logopts + LogOpts = get_start_opt(logopts, value, [], StartOpts), + %% config & userconfig CfgFiles = ct_config:get_config_file_list(StartOpts), @@ -768,8 +832,9 @@ run_test1(StartOpts) -> %% stepped execution Step = get_start_opt(step, value, StartOpts), - Opts = #opts{label = Label, - cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, + Opts = #opts{label = Label, profile = Profile, + cover = Cover, step = Step, logdir = LogDir, + logopts = LogOpts, config = CfgFiles, event_handlers = EvHandlers, ct_hooks = CTHooks, enable_builtin_hooks = EnableBuiltinHooks, @@ -811,8 +876,12 @@ run_spec_file(Relaxed, SpecOpts = get_data_for_node(TS, node()), Label = choose_val(Opts#opts.label, SpecOpts#opts.label), + Profile = choose_val(Opts#opts.profile, + SpecOpts#opts.profile), LogDir = choose_val(Opts#opts.logdir, SpecOpts#opts.logdir), + AllLogOpts = merge_vals([Opts#opts.logopts, + SpecOpts#opts.logopts]), AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), Cover = choose_val(Opts#opts.cover, SpecOpts#opts.cover), @@ -833,8 +902,10 @@ run_spec_file(Relaxed, application:set_env(common_test, include, AllInclude), Opts1 = Opts#opts{label = Label, + profile = Profile, cover = Cover, logdir = which(logdir, LogDir), + logopts = AllLogOpts, config = AllConfig, event_handlers = AllEvHs, include = AllInclude, @@ -848,7 +919,6 @@ run_spec_file(Relaxed, case check_and_install_configfiles(AllConfig,Opts1#opts.logdir, Opts1) of ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); {error,GCFReason} -> @@ -921,67 +991,102 @@ run_dir(Opts = #opts{logdir = LogDir, ok -> ok; {error,IReason} -> exit(IReason) end, - case lists:keysearch(dir, 1, StartOpts) of - {value,{_,Dirs=[Dir|_]}} when not is_integer(Dir), - length(Dirs)>1 -> - %% multiple dirs (no suite) - reformat_result(catch do_run(tests(Dirs), [], Opts1, StartOpts)); - false -> % no dir - %% fun for converting suite name to {Dir,Mod} tuple - S2M = fun(S) when is_list(S) -> - {filename:dirname(S), - list_to_atom(filename:rootname(filename:basename(S)))}; - (A) -> - {".",A} - end, - case lists:keysearch(suite, 1, StartOpts) of - {value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) -> - {Dir,Mod} = S2M(Suite), - case groups_and_cases(proplists:get_value(group, StartOpts), - proplists:get_value(testcase, StartOpts)) of - Error = {error,_} -> - exit(Error); + case {proplists:get_value(dir, StartOpts), + proplists:get_value(suite, StartOpts), + groups_and_cases(proplists:get_value(group, StartOpts), + proplists:get_value(testcase, StartOpts))} of + %% flag specified without data + {_,_,Error={error,_}} -> + Error; + {_,[],_} -> + {error,no_suite_specified}; + {[],_,_} -> + {error,no_dir_specified}; + + {Dirs=[Hd|_],undefined,[]} when is_list(Dirs), not is_integer(Hd) -> + Dirs1 = [if is_atom(D) -> atom_to_list(D); + true -> D end || D <- Dirs], + reformat_result(catch do_run(tests(Dirs1), [], Opts1, StartOpts)); + + {Dir=[Hd|_],undefined,[]} when is_list(Dir) and is_integer(Hd) -> + reformat_result(catch do_run(tests(Dir), [], Opts1, StartOpts)); + + {Dir,undefined,[]} when is_atom(Dir) and (Dir /= undefined) -> + reformat_result(catch do_run(tests(atom_to_list(Dir)), + [], Opts1, StartOpts)); + + {undefined,Suites=[Hd|_],[]} when not is_integer(Hd) -> + Suites1 = [suite_to_test(S) || S <- Suites], + reformat_result(catch do_run(tests(Suites1), [], Opts1, StartOpts)); + + {undefined,Suite,[]} when is_atom(Suite) and + (Suite /= undefined) -> + {Dir,Mod} = suite_to_test(Suite), + reformat_result(catch do_run(tests(Dir, Mod), [], Opts1, StartOpts)); + + {undefined,Suite,GsAndCs} when is_atom(Suite) and + (Suite /= undefined) -> + {Dir,Mod} = suite_to_test(Suite), + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)); + + {undefined,[Hd,_|_],_GsAndCs} when not is_integer(Hd) -> + exit(multiple_suites_and_cases); + + {undefined,Suite=[Hd|Tl],GsAndCs} when is_integer(Hd) ; + (is_list(Hd) and (Tl == [])) ; + (is_atom(Hd) and (Tl == [])) -> + {Dir,Mod} = suite_to_test(Suite), + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)); + + {[Hd,_|_],_Suites,[]} when is_list(Hd) ; not is_integer(Hd) -> + exit(multiple_dirs_and_suites); + + {undefined,undefined,GsAndCs} when GsAndCs /= [] -> + exit(incorrect_start_options); + + {Dir,Suite,GsAndCs} when is_integer(hd(Dir)) ; + (is_atom(Dir) and (Dir /= undefined)) ; + ((length(Dir) == 1) and is_atom(hd(Dir))) ; + ((length(Dir) == 1) and is_list(hd(Dir))) -> + Dir1 = if is_atom(Dir) -> atom_to_list(Dir); + true -> Dir end, + if Suite == undefined -> + exit(incorrect_start_options); + + is_integer(hd(Suite)) ; + (is_atom(Suite) and (Suite /= undefined)) ; + ((length(Suite) == 1) and is_atom(hd(Suite))) ; + ((length(Suite) == 1) and is_list(hd(Suite))) -> + {Dir2,Mod} = suite_to_test(Dir1, Suite), + case GsAndCs of [] -> - reformat_result(catch do_run(tests(Dir, listify(Mod)), + reformat_result(catch do_run(tests(Dir2, Mod), [], Opts1, StartOpts)); - GsAndCs -> - reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + _ -> + reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), [], Opts1, StartOpts)) end; - {value,{_,Suites}} -> - reformat_result(catch do_run(tests(lists:map(S2M, Suites)), - [], Opts1, StartOpts)); - _ -> - exit(no_tests_specified) - end; - {value,{_,Dir}} -> - case lists:keysearch(suite, 1, StartOpts) of - {value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) -> - Mod = if is_atom(Suite) -> Suite; - true -> list_to_atom(Suite) - end, - case groups_and_cases(proplists:get_value(group, StartOpts), - proplists:get_value(testcase, StartOpts)) of - Error = {error,_} -> - exit(Error); - [] -> - reformat_result(catch do_run(tests(Dir, listify(Mod)), + + is_list(Suite) -> % multiple suites + case [suite_to_test(Dir1, S) || S <- Suite] of + [_,_|_] when GsAndCs /= [] -> + exit(multiple_suites_and_cases); + [{Dir2,Mod}] when GsAndCs /= [] -> + reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), [], Opts1, StartOpts)); - GsAndCs -> - reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + DirMods -> + reformat_result(catch do_run(tests(DirMods), [], Opts1, StartOpts)) - end; - {value,{_,Suites=[Suite|_]}} when is_list(Suite) -> - Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites), - reformat_result(catch do_run(tests(delistify(Dir), Mods), - [], Opts1, StartOpts)); - {value,{_,Suites}} -> - reformat_result(catch do_run(tests(delistify(Dir), Suites), - [], Opts1, StartOpts)); - false -> % no suite, only dir - reformat_result(catch do_run(tests(listify(Dir)), - [], Opts1, StartOpts)) - end + end + end; + + {undefined,undefined,[]} -> + exit(no_test_specified); + + {Dir,Suite,GsAndCs} -> + exit({incorrect_start_options,{Dir,Suite,GsAndCs}}) end. %%%----------------------------------------------------------------- @@ -992,19 +1097,38 @@ run_dir(Opts = #opts{logdir = LogDir, %%% the same as those used in test specification files. %%% @equiv ct:run_testspec/1 %%%----------------------------------------------------------------- - run_testspec(TestSpec) -> + CTPid = spawn(fun() -> run_testspec1(TestSpec) end), + Ref = monitor(process, CTPid), + receive + {'DOWN',Ref,process,CTPid,{user_error,Error}} -> + Error; + {'DOWN',Ref,process,CTPid,Other} -> + Other + end. + +run_testspec1(TestSpec) -> {ok,Cwd} = file:get_cwd(), io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]), - case catch run_testspec1(TestSpec) of + case catch run_testspec2(TestSpec) of {'EXIT',Reason} -> file:set_cwd(Cwd), - {error,Reason}; + exit({error,Reason}); Result -> - Result + exit(Result) end. -run_testspec1(TestSpec) -> +run_testspec2(File) when is_list(File), is_integer(hd(File)) -> + case file:read_file_info(File) of + {ok,_} -> + exit("Bad argument, " + "use ct:run_test([{spec," ++ File ++ "}])"); + _ -> + exit("Bad argument, list of tuples expected, " + "use ct:run_test/1 for test specification files") + end; + +run_testspec2(TestSpec) -> case catch ct_testspec:collect_tests_from_list(TestSpec, false) of {E,CTReason} when E == error ; E == 'EXIT' -> exit(CTReason); @@ -1035,7 +1159,9 @@ run_testspec1(TestSpec) -> end. get_data_for_node(#testspec{label = Labels, + profile = Profiles, logdir = LogDirs, + logopts = LogOptsList, cover = CoverFs, config = Cfgs, userconfig = UsrCfgs, @@ -1046,10 +1172,15 @@ get_data_for_node(#testspec{label = Labels, multiply_timetraps = MTs, scale_timetraps = STs}, Node) -> Label = proplists:get_value(Node, Labels), + Profile = proplists:get_value(Node, Profiles), LogDir = case proplists:get_value(Node, LogDirs) of undefined -> "."; Dir -> Dir end, + LogOpts = case proplists:get_value(Node, LogOptsList) of + undefined -> []; + LOs -> LOs + end, Cover = proplists:get_value(Node, CoverFs), MT = proplists:get_value(Node, MTs), ST = proplists:get_value(Node, STs), @@ -1059,7 +1190,9 @@ get_data_for_node(#testspec{label = Labels, FiltCTHooks = [Hook || {N,Hook} <- CTHooks, N==Node], Include = [I || {N,I} <- Incl, N==Node], #opts{label = Label, + profile = Profile, logdir = LogDir, + logopts = LogOpts, cover = Cover, config = ConfigFiles, event_handlers = EvHandlers, @@ -1141,8 +1274,24 @@ reformat_result({user_error,Reason}) -> reformat_result(Result) -> Result. -suite_to_test(Suite) -> - {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}. +suite_to_test(Suite) when is_atom(Suite) -> + suite_to_test(atom_to_list(Suite)); + +suite_to_test(Suite) when is_list(Suite) -> + {filename:dirname(Suite), + list_to_atom(filename:rootname(filename:basename(Suite)))}. + +suite_to_test(Dir, Suite) when is_atom(Suite) -> + suite_to_test(Dir, atom_to_list(Suite)); + +suite_to_test(Dir, Suite) when is_list(Suite) -> + case filename:dirname(Suite) of + "." -> + {Dir,list_to_atom(filename:rootname(Suite))}; + DirName -> % ignore Dir + File = filename:basename(Suite), + {DirName,list_to_atom(filename:rootname(File))} + end. groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and ((Cs == undefined) or (Cs == [])) -> @@ -1176,9 +1325,11 @@ tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) -> [{?testdir(TestDir,all),all,all} || TestDir <- TestDirs]. do_run(Tests, Misc) when is_list(Misc) -> - do_run(Tests, Misc, "."). + do_run(Tests, Misc, ".", []). -do_run(Tests, Misc, LogDir) when is_list(Misc) -> +do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), + is_list(LogDir), + is_list(LogOpts) -> Opts = case proplists:get_value(step, Misc) of undefined -> @@ -1193,11 +1344,10 @@ do_run(Tests, Misc, LogDir) when is_list(Misc) -> CoverFile -> Opts#opts{cover = CoverFile} end, - do_run(Tests, [], Opts1#opts{logdir = LogDir}, []). - -do_run(Tests, Skip, Opts, Args) -> - #opts{label = Label, cover = Cover} = Opts, + do_run(Tests, [], Opts1#opts{logdir = LogDir}, []); +do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> + #opts{label = Label, profile = Profile, cover = Cover} = Opts, %% label - used by ct_logs TestLabel = if Label == undefined -> undefined; @@ -1207,6 +1357,15 @@ do_run(Tests, Skip, Opts, Args) -> end, application:set_env(common_test, test_label, TestLabel), + %% profile - used in ct_util + TestProfile = + if Profile == undefined -> undefined; + is_atom(Profile) -> atom_to_list(Profile); + is_list(Profile) -> Profile; + true -> undefined + end, + application:set_env(common_test, profile, TestProfile), + case code:which(test_server) of non_existing -> exit({error,no_path_to_test_server}); @@ -1241,6 +1400,8 @@ do_run(Tests, Skip, Opts, Args) -> _Pid -> %% save stylesheet info ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), + %% save logopts + ct_util:set_testdata({logopts,Opts#opts.logopts}), %% enable silent connections case Opts#opts.silent_connections of [] -> @@ -2008,7 +2169,14 @@ maybe_interpret1(Suite, Cases, StepOpts) when is_list(Cases) -> maybe_interpret2(Suite, Cases, StepOpts) -> set_break_on_config(Suite, StepOpts), - [i:ib(Suite, Case, 1) || Case <- Cases], + [begin try i:ib(Suite, Case, 1) of + _ -> ok + catch + _:_Error -> + io:format(user, "Invalid breakpoint: ~w:~w/1~n", + [Suite,Case]) + end + end || Case <- Cases, is_atom(Case)], test_server_ctrl:multiply_timetraps(infinity), WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of true -> no_kill; @@ -2021,10 +2189,18 @@ maybe_interpret2(Suite, Cases, StepOpts) -> set_break_on_config(Suite, StepOpts) -> case lists:member(config, ensure_atom(StepOpts)) of true -> - i:ib(Suite, init_per_suite, 1), - i:ib(Suite, init_per_testcase, 2), - i:ib(Suite, end_per_testcase, 2), - i:ib(Suite, end_per_suite, 1); + SetBPIfExists = fun(F,A) -> + case erlang:function_exported(Suite, F, A) of + true -> i:ib(Suite, F, A); + false -> ok + end + end, + SetBPIfExists(init_per_suite, 1), + SetBPIfExists(init_per_group, 2), + SetBPIfExists(init_per_testcase, 2), + SetBPIfExists(end_per_testcase, 2), + SetBPIfExists(end_per_group, 2), + SetBPIfExists(end_per_suite, 1); false -> ok end. @@ -2078,6 +2254,15 @@ get_start_opt(Key, IfExists, Args) -> get_start_opt(Key, IfExists, undefined, Args). get_start_opt(Key, IfExists, IfNotExists, Args) -> + try try_get_start_opt(Key, IfExists, IfNotExists, Args) of + Result -> + Result + catch + error:_ -> + exit({user_error,{bad_argument,Key}}) + end. + +try_get_start_opt(Key, IfExists, IfNotExists, Args) -> case lists:keysearch(Key, 1, Args) of {value,{Key,Val}} when is_function(IfExists) -> IfExists(Val); @@ -2252,6 +2437,8 @@ opts2args(EnvStartOpts) -> end, EHs), [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), [{event_handler_init,lists:reverse(StrsR)}]; + ({logopts,LOs}) when is_list(LOs) -> + [{logopts,[atom_to_list(LO) || LO <- LOs]}]; ({ct_hooks,[]}) -> []; ({ct_hooks,CTHs}) when is_list(CTHs) -> @@ -2315,32 +2502,31 @@ is_suite(ModOrFile) when is_list(ModOrFile) -> end. get_all_testcases(Suite) -> - %%! this needs to be updated to handle testcase groups later!! - case catch Suite:all() of - {'EXIT',Why} -> - {error,Why}; - {skip,_} -> - []; - Cases -> - AllCases = - lists:foldl(fun({sequence,SeqName}, All) -> - case catch Suite:sequences() of - {'EXIT',_} -> - All; - Seqs -> - case proplists:get_value(SeqName, Seqs) of - undefined -> - All; - SeqCases -> - lists:reverse(SeqCases) ++ All - end - end; - (Case,All) -> - [Case|All] - end, [], Cases), - lists:reverse(AllCases) + try ct_framework:get_all_cases(Suite) of + {error,_Reason} = Error -> + Error; + SuiteCases -> + Cases = [C || {_S,C} <- SuiteCases], + try Suite:sequences() of + [] -> + Cases; + Seqs -> + TCs1 = lists:flatten([TCs || {_,TCs} <- Seqs]), + lists:reverse( + lists:foldl(fun(TC, Acc) -> + case lists:member(TC, Acc) of + true -> Acc; + false -> [TC | Acc] + end + end, [], Cases ++ TCs1)) + catch + _:_ -> + Cases + end + catch + _:Error -> + {error,Error} end. - %% Internal tracing support. If {ct_trace,TraceSpec} is present, the %% TraceSpec file will be consulted and dbg used to trace function @@ -2361,8 +2547,8 @@ start_trace(Args) -> false end; {_,Error} -> - io:format("Warning! Tracing not started. Reason: ~p~n~n", - [Error]), + io:format("Warning! Tracing not started. Reason: ~s~n~n", + [file:format_error(Error)]), false end; false -> diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index c6f5fd7df4..71a784870c 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -245,7 +245,6 @@ cmdf(Connection,CmdFormat,Args) -> %%% Data = [string()] %%% @doc Send a telnet command and wait for prompt %%% (uses a format string and list of arguments to build the command). -%%%----------------------------------------------------------------- cmdf(Connection,CmdFormat,Args,Timeout) when is_list(Args) -> Cmd = lists:flatten(io_lib:format(CmdFormat,Args)), cmd(Connection,Cmd,Timeout). @@ -360,15 +359,15 @@ expect(Connection,Patterns) -> %%% will also be a <code>HaltReason</code> returned.</p> %%% %%% <p><underline>Examples:</underline><br/> -%%% <code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}], -%%% [sequence,{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match +%%% <code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}],</code> +%%% <code>[sequence,{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match %%% "ABC" first and then "XYZ", but if "NNN" appears the function will %%% return <code>{error,{nnn,["NNN"]}}</code>. If both "ABC" and "XYZ" %%% are matched, the function will return %%% <code>{ok,[AbcMatch,XyzMatch]}</code>.</p> %%% -%%% <p><code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}], -%%% [{repeat,2},{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match +%%% <p><code>expect(Connection,[{abc,"ABC"},{xyz,"XYZ"}],</code> +%%% <code>[{repeat,2},{halt,[{nnn,"NNN"}]}]).</code><br/> will try to match %%% "ABC" or "XYZ" twice. If "NNN" appears the function will return %%% with <code>HaltReason = {nnn,["NNN"]}</code>.</p> %%% diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 96971ccfc7..317910d5c8 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -249,11 +249,15 @@ collect_tests_from_file1([Spec|Specs],TestSpec,Relaxed) -> SpecDir = filename:dirname(filename:absname(Spec)), case file:consult(Spec) of {ok,Terms} -> - TestSpec1 = collect_tests(Terms,TestSpec#testspec{spec_dir=SpecDir}, + TestSpec1 = collect_tests(Terms, + TestSpec#testspec{spec_dir=SpecDir}, Relaxed), collect_tests_from_file1(Specs,TestSpec1,Relaxed); {error,Reason} -> - throw({error,{Spec,Reason}}) + ReasonStr = + lists:flatten(io_lib:format("~s", + [file:format_error(Reason)])), + throw({error,{Spec,ReasonStr}}) end; collect_tests_from_file1([],TS=#testspec{config=Cfgs,event_handler=EvHs, include=Incl,tests=Tests},_) -> @@ -481,6 +485,26 @@ add_tests([{logdir,Node,Dir}|Ts],Spec) -> add_tests([{logdir,Dir}|Ts],Spec) -> add_tests([{logdir,all_nodes,Dir}|Ts],Spec); +%% --- logopts --- +add_tests([{logopts,all_nodes,Opts}|Ts],Spec) -> + LogOpts = Spec#testspec.logopts, + Tests = [{logopts,N,Opts} || + N <- list_nodes(Spec), + lists:keymember(ref2node(N,Spec#testspec.nodes),1, + LogOpts) == false], + add_tests(Tests++Ts,Spec); +add_tests([{logopts,Nodes,Opts}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,logopts,[Opts],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{logopts,Node,Opts}|Ts],Spec) -> + LogOpts = Spec#testspec.logopts, + LogOpts1 = [{ref2node(Node,Spec#testspec.nodes),Opts} | + lists:keydelete(ref2node(Node,Spec#testspec.nodes), + 1,LogOpts)], + add_tests(Ts,Spec#testspec{logopts=LogOpts1}); +add_tests([{logopts,Opts}|Ts],Spec) -> + add_tests([{logopts,all_nodes,Opts}|Ts],Spec); + %% --- label --- add_tests([{label,all_nodes,Lbl}|Ts],Spec) -> Labels = Spec#testspec.label, @@ -1101,6 +1125,8 @@ valid_terms() -> {merge_tests,1}, {logdir,2}, {logdir,3}, + {logopts,2}, + {logopts,3}, {label,2}, {label,3}, {event_handler,2}, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index b3e345b4e5..3b6ad6f98d 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -47,7 +47,7 @@ -export([get_mode/0, create_table/3, read_opts/0]). --export([set_cwd/1, reset_cwd/0]). +-export([set_cwd/1, reset_cwd/0, get_start_dir/0]). -export([parse_table/1]). @@ -61,6 +61,9 @@ -export([warn_duplicates/1]). +-export([get_profile_data/0, get_profile_data/1, + get_profile_data/2, open_url/3]). + -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -121,13 +124,15 @@ do_start(Parent,Mode,LogDir) -> ok -> ok; E -> exit(E) end, + DoExit = fun(Reason) -> file:set_cwd(StartDir), exit(Reason) end, Opts = case read_opts() of {ok,Opts1} -> Opts1; Error -> Parent ! {self(),Error}, - exit(Error) + DoExit(Error) end, + %% start an event manager (if not already started by master) case ct_event:start_link() of {error,{already_started,_}} -> @@ -140,16 +145,23 @@ do_start(Parent,Mode,LogDir) -> ct_event:add_handler([{vts,VtsPid}]) end end, + %% start ct_config server - ct_config:start(Mode), + try ct_config:start(Mode) of + _ -> ok + catch + _Class:CfgError -> + DoExit(CfgError) + end, + %% add user event handlers case lists:keysearch(event_handler,1,Opts) of {value,{_,Handlers}} -> Add = fun({H,Args}) -> case catch gen_event:add_handler(?CT_EVMGR_REF,H,Args) of ok -> ok; - {'EXIT',Why} -> exit(Why); - Other -> exit({event_handler,Other}) + {'EXIT',Why} -> DoExit(Why); + Other -> DoExit({event_handler,Other}) end end, case catch lists:foreach(Add,Handlers) of @@ -168,10 +180,15 @@ do_start(Parent,Mode,LogDir) -> data={StartTime, lists:flatten(TestLogDir)}}), %% Initialize ct_hooks - case catch ct_hooks:init(Opts) of + try ct_hooks:init(Opts) of ok -> Parent ! {self(),started}; - {_,CTHReason} -> + {fail,CTHReason} -> + ct_logs:tc_print('Suite Callback',CTHReason,[]), + self() ! {{stop,{self(),{user_error,CTHReason}}}, + {Parent,make_ref()}} + catch + _:CTHReason -> ct_logs:tc_print('Suite Callback',CTHReason,[]), self() ! {{stop,{self(),{user_error,CTHReason}}}, {Parent,make_ref()}} @@ -243,6 +260,9 @@ set_cwd(Dir) -> reset_cwd() -> call(reset_cwd). +get_start_dir() -> + call(get_start_dir). + loop(Mode,TestData,StartDir) -> receive {update_last_run_index,From} -> @@ -319,6 +339,9 @@ loop(Mode,TestData,StartDir) -> {reset_cwd,From} -> return(From,file:set_cwd(StartDir)), loop(From,TestData,StartDir); + {get_start_dir,From} -> + return(From,StartDir), + loop(From,TestData,StartDir); {{stop,Info},From} -> Time = calendar:local_time(), ct_event:sync_notify(#event{name=test_done, @@ -332,7 +355,7 @@ loop(Mode,TestData,StartDir) -> ets:delete(?conn_table), ets:delete(?board_table), ets:delete(?suite_table), - ct_logs:close(Info), + ct_logs:close(Info, StartDir), ct_event:stop(), ct_config:stop(), file:set_cwd(StartDir), @@ -727,6 +750,79 @@ warn_duplicates(Suites) -> lists:foreach(Warn, Suites), ok. +%%%----------------------------------------------------------------- +%%% @spec +%%% +%%% @doc +get_profile_data() -> + get_profile_data(all). + +get_profile_data(KeyOrStartDir) -> + if is_atom(KeyOrStartDir) -> + get_profile_data(KeyOrStartDir, get_start_dir()); + is_list(KeyOrStartDir) -> + get_profile_data(all, KeyOrStartDir) + end. + +get_profile_data(Key, StartDir) -> + Profile = case application:get_env(common_test, profile) of + {ok,undefined} -> default; + {ok,Prof} -> Prof; + _ -> default + end, + get_profile_data(Profile, Key, StartDir). + +get_profile_data(Profile, Key, StartDir) -> + File = case Profile of + default -> + ?ct_profile_file; + _ when is_list(Profile) -> + ?ct_profile_file ++ "." ++ Profile; + _ when is_atom(Profile) -> + ?ct_profile_file ++ "." ++ atom_to_list(Profile) + end, + FullNameWD = filename:join(StartDir, File), + {WhichFile,Result} = + case file:consult(FullNameWD) of + {error,enoent} -> + case init:get_argument(home) of + {ok,[[HomeDir]]} -> + FullNameHome = filename:join(HomeDir, File), + {FullNameHome,file:consult(FullNameHome)}; + _ -> + {File,{error,enoent}} + end; + Consulted -> + {FullNameWD,Consulted} + end, + case Result of + {error,enoent} when Profile /= default -> + io:format(user, "~nERROR! Missing profile file ~p~n", [File]), + undefined; + {error,enoent} when Profile == default -> + undefined; + {error,Reason} -> + io:format(user,"~nERROR! Error in profile file ~p: ~p~n", + [WhichFile,Reason]), + undefined; + {ok,Data} -> + Data1 = case Data of + [List] when is_list(List) -> + List; + _ when is_list(Data) -> + Data; + _ -> + io:format(user, + "~nERROR! Invalid profile data in ~p~n", + [WhichFile]), + [] + end, + if Key == all -> + Data1; + true -> + proplists:get_value(Key, Data) + end + end. %%%----------------------------------------------------------------- %%% Internal functions @@ -799,3 +895,28 @@ abs_name2([H|T],Acc) -> abs_name2(T,[H|Acc]); abs_name2([],Acc) -> filename:join(lists:reverse(Acc)). + +open_url(iexplore, Args, URL) -> + {ok,R} = win32reg:open([read]), + ok = win32reg:change_key(R,"applications\\iexplore.exe\\shell\\open\\command"), + case win32reg:values(R) of + {ok, Paths} -> + Path = proplists:get_value(default, Paths), + [Cmd | _] = string:tokens(Path, "%"), + Cmd1 = Cmd ++ " " ++ Args ++ " " ++ URL, + io:format(user, "~nOpening ~s with command:~n ~s~n", [URL,Cmd1]), + open_port({spawn,Cmd1}, []); + _ -> + io:format("~nNo path to iexplore.exe~n",[]) + end, + win32reg:close(R), + ok; + +open_url(Prog, Args, URL) -> + ProgStr = if is_atom(Prog) -> atom_to_list(Prog); + is_list(Prog) -> Prog + end, + Cmd = ProgStr ++ " " ++ Args ++ " " ++ URL, + io:format(user, "~nOpening ~s with command:~n ~s~n", [URL,Cmd]), + open_port({spawn,Cmd},[]), + ok. diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index dbe9d9b4e6..bde832811a 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -31,7 +31,9 @@ nodes=[], init=[], label=[], + profile=[], logdir=["."], + logopts=[], cover=[], config=[], userconfig=[], @@ -59,3 +61,5 @@ -define(missing_suites_info, "missing_suites.info"). -define(ct_config_txt, ct_config_plain). + +-define(ct_profile_file, ".common_test"). diff --git a/lib/common_test/src/vts.erl b/lib/common_test/src/vts.erl index f0bf090804..cc8a932887 100644 --- a/lib/common_test/src/vts.erl +++ b/lib/common_test/src/vts.erl @@ -20,7 +20,7 @@ -module(vts). -export([start/0, - init_data/4, + init_data/5, stop/0, report/2]). @@ -32,6 +32,7 @@ menu_frame/2, welcome_frame/2, config_frame/2, + browse_config_file/2, add_config_file/2, remove_config_file/2, run_frame/2, @@ -56,7 +57,7 @@ -record(state,{tests=[],config=[],event_handler=[],test_runner, running=0,reload_results=false,start_dir,current_log_dir, - total=0,ok=0,fail=0,skip=0,testruns=[]}). + logopts=[],total=0,ok=0,fail=0,skip=0,testruns=[]}). %%%----------------------------------------------------------------- @@ -65,8 +66,8 @@ start() -> webtool:start(), webtool:start_tools([],"app=vts"). -init_data(ConfigFiles,EvHandlers,LogDir,Tests) -> - call({init_data,ConfigFiles,EvHandlers,LogDir,Tests}). +init_data(ConfigFiles,EvHandlers,LogDir,LogOpts,Tests) -> + call({init_data,ConfigFiles,EvHandlers,LogDir,LogOpts,Tests}). stop() -> webtool:stop_tools([],"app=vts"), @@ -119,6 +120,8 @@ menu_frame(_Env,_Input) -> call(menu_frame). config_frame(_Env,_Input) -> call(config_frame). +browse_config_file(_Env,Input) -> + call({browse_config_file,Input}). add_config_file(_Env,Input) -> call({add_config_file,Input}). remove_config_file(_Env,Input) -> @@ -160,10 +163,11 @@ init(Parent) -> loop(State) -> receive - {{init_data,Config,EvHandlers,LogDir,Tests},From} -> + {{init_data,Config,EvHandlers,LogDir,LogOpts,Tests},From} -> %% ct:pal("State#state.current_log_dir=~p", [State#state.current_log_dir]), NewState = State#state{config=Config,event_handler=EvHandlers, - current_log_dir=LogDir,tests=Tests}, + current_log_dir=LogDir, + logopts=LogOpts,tests=Tests}, ct_install(NewState), return(From,ok), loop(NewState); @@ -182,6 +186,9 @@ loop(State) -> {config_frame,From} -> return(From,config_frame1(State)), loop(State); + {{browse_config_file,_Input},From} -> + return(From,ok), + loop(State); {{add_config_file,Input},From} -> {Return,State1} = add_config_file1(Input,State), ct_install(State1), @@ -241,10 +248,12 @@ loop(State) -> return(From,ok); {'EXIT',Pid,Reason} -> case State#state.test_runner of - Pid -> io:format("ERROR: test runner crashed: ~p\n",[Reason]); - _ -> ignore - end, - loop(State); + Pid -> + io:format("Test run error: ~p\n",[Reason]), + loop(State); + _ -> + loop(State) + end; {{test_info,_Type,_Data},From} -> return(From,ok), loop(State) @@ -270,10 +279,11 @@ return({To,Ref},Result) -> To ! {Ref, Result}. -run_test1(State=#state{tests=Tests,current_log_dir=LogDir}) -> +run_test1(State=#state{tests=Tests,current_log_dir=LogDir, + logopts=LogOpts}) -> Self=self(), RunTest = fun() -> - case ct_run:do_run(Tests,[],LogDir) of + case ct_run:do_run(Tests,[],LogDir,LogOpts) of {error,_Reason} -> aborted(); _ -> @@ -282,17 +292,18 @@ run_test1(State=#state{tests=Tests,current_log_dir=LogDir}) -> unlink(Self) end, Pid = spawn_link(RunTest), - Total = + {Total,Tests1} = receive {{test_info,start_info,{_,_,Cases}},From} -> return(From,ok), - Cases; + {Cases,Tests}; EXIT = {'EXIT',_,_} -> - self() ! EXIT + self() ! EXIT, + {0,[]} after 30000 -> - 0 + {0,[]} end, - State#state{test_runner=Pid,running=length(Tests), + State#state{test_runner=Pid,running=length(Tests1), total=Total,ok=0,fail=0,skip=0,testruns=[]}. @@ -356,22 +367,32 @@ config_frame1(State) -> config_body(State) -> Entry = [input("TYPE=file NAME=browse SIZE=40"), input("TYPE=hidden NAME=file")], + BrowseForm = + form( + "NAME=read_file_form METHOD=post ACTION=\"./browse_config_file\"", + table( + "BORDER=0", + [tr(td("1. Locate config file")), + tr(td(Entry))])), AddForm = form( - "NAME=read_file_form METHOD=post ACTION=\"./add_config_file\"", + "NAME=add_file_form METHOD=post ACTION=\"./add_config_file\"", table( "BORDER=0", - [tr( - [td(Entry), + [tr(td("2. Paste full config file name here")), + tr( + [td(input("TYPE=text NAME=file SIZE=40")), td("ALIGN=center", input("TYPE=submit onClick=\"file.value=browse.value;\"" " VALUE=\"Add\""))])])), + {Text,RemoveForm} = case State#state.config of [] -> - T = "To be able to run any tests, one or more configuration " - "files must be added. Enter the name of the configuration " - "file below and click the \"Add\" button.", + T = "Before running the tests, one or more configuration " + "files may be added. Locate the config file, copy its " + "full name, paste this into the text field below, then " + "click the \"Add\" button.", R = "", {T,R}; Files -> @@ -394,20 +415,24 @@ config_body(State) -> input("TYPE=submit VALUE=\"Remove\"")))])), {T,R} end, - + [h1("ALIGN=center","Config"), table( - "WIDTH=600 ALIGN=center CELLPADDING=5", + "WIDTH=450 ALIGN=center CELLPADDING=5", [tr(td(["BGCOLOR=",?INFO_BG_COLOR],Text)), - tr(td("ALIGN=center",AddForm)), - tr(td("ALIGN=center",RemoveForm))])]. - + tr(td("")), + tr(td("")), + tr(td("ALIGN=left",BrowseForm)), + tr(td("ALIGN=left",AddForm)), + tr(td("ALIGN=left",RemoveForm))])]. add_config_file1(Input,State) -> State1 = case get_input_data(Input,"file") of - "" -> State; - File -> State#state{config=[File|State#state.config]} + "" -> + State; + File -> + State#state{config=[File|State#state.config]} end, Return = config_frame1(State1), {Return,State1}. @@ -427,10 +452,17 @@ run_body(#state{running=Running}) when Running>0 -> [h1("ALIGN=center","Run Test"), p(["Test are ongoing: ",href("./result_frameset","Results")])]; run_body(State) -> - ConfigList = ul([li(File) || File <- State#state.config]), + ConfigList = + case State#state.config of + [] -> + ul(["none"]); + CfgFiles -> + ul([li(File) || File <- CfgFiles]) + end, ConfigFiles = [h3("Config Files"), ConfigList], - + {ok,CWD} = file:get_cwd(), + CurrWD = [h3("Current Working Directory"), ul(CWD)], AddDirForm = form( "NAME=add_dir_form METHOD=post ACTION=\"./add_test_dir\"", @@ -442,7 +474,6 @@ run_body(State) -> td("ALIGN=center", input("TYPE=submit onClick=\"dir.value=browse.value;\"" " VALUE=\"Add Test Dir\""))])])), - {LoadedTestsTable,Submit} = case create_testdir_entries(State#state.tests,1) of [] -> {"",""}; @@ -454,22 +485,20 @@ run_body(State) -> {table("CELLPADDING=5",[Heading,TestDirs]), submit_button()} end, - - %% It should be ok to have no config-file! Body = - %% case State#state.config of %% [] -> %% p("ALIGN=center", - %% href("./config_frame","Please select one or - %% more config files")); %% _ -> table( - "WIDTH=100%", - [tr(td(ConfigFiles)), + "WIDTH=450 ALIGN=center", + [tr(td("")), + tr(td("")), + tr(td(ConfigFiles)), + tr(td("")), + tr(td(CurrWD)), tr(td("")), tr(td(AddDirForm)), tr(td("")), tr(td(LoadedTestsTable)), - tr(td(Submit))]), - %% end, - + tr(td(Submit)) + ]), [h1("ALIGN=center","Run Test"), Body]. create_testdir_entries([{Dir,Suite,Case}|Tests],N) -> @@ -556,18 +585,17 @@ options([Element|Elements],Selected,N,Func) -> options([],_Selected,_N,_Func) -> []. -add_test_dir1(Input,State) -> +add_test_dir1(Input, State) -> State1 = case get_input_data(Input,"dir") of "" -> State; Dir0 -> Dir = case ct_util:is_test_dir(Dir0) of - true -> - Dir0; - false -> filename:join(Dir0,"test") + true -> Dir0; + false -> ct_util:get_testdir(Dir0, all) end, case filelib:is_dir(Dir) of - true -> + true -> Test = ct_run:tests(Dir), State#state{tests=State#state.tests++Test}; false -> @@ -577,8 +605,6 @@ add_test_dir1(Input,State) -> Return = run_frame1(State1), {Return,State1}. - - remove_test_dir1(Input,State) -> N = list_to_integer(get_input_data(Input,"dir")), State1 = State#state{tests=delete_test(N,State#state.tests)}, @@ -641,6 +667,9 @@ result_frameset2(State) -> "./redirect_to_result_log_frame"; {_Dir,0} -> filename:join(["/log_dir","index.html"]); + {_Dir,_} when State#state.testruns == [] -> + %% crash before first test + "./no_result_log_frame"; {_Dir,_} -> {_,CurrentLog} = hd(State#state.testruns), CurrentLog @@ -749,6 +778,8 @@ report1(tc_done,{_Suite,_Case,{skipped,_Reason}},State) -> State#state{skip=State#state.skip+1}; report1(tc_user_skip,{_Suite,_Case,_Reason},State) -> State#state{skip=State#state.skip+1}; +report1(tc_auto_skip,{_Suite,_Case,_Reason},State) -> + State#state{skip=State#state.skip+1}; report1(loginfo,_,State) -> State. @@ -850,6 +881,8 @@ h2(Text) -> ["<H2>",Text,"</H2>\n"]. h3(Text) -> ["<H3>",Text,"</H3>\n"]. +%%h4(Text) -> +%% ["<H4>",Text,"</H4>\n"]. font(Args,Text) -> ["<FONT ",Args,">\n",Text,"\n</FONT>\n"]. p(Text) -> diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 836443009f..c1a455c6d8 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -60,7 +60,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, - timetrap_normal, timetrap_extended]. + timetrap_normal, timetrap_extended, timetrap_parallel, + timetrap_fun]. groups() -> []. @@ -228,6 +229,28 @@ timetrap_parallel(Config) when is_list(Config) -> ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- +%%% +timetrap_fun(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_4_SUITE"), + Join(DataDir, "timetrap_5_SUITE"), + Join(DataDir, "timetrap_6_SUITE"), + Join(DataDir, "timetrap_7_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_fun, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(timetrap_fun), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -260,7 +283,7 @@ test_events(cfg_error) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{14,14,43}}, + {?eh,start_info,{14,14,45}}, {?eh,tc_start,{cfg_error_1_SUITE,init_per_suite}}, {?eh,tc_done, @@ -517,13 +540,18 @@ test_events(cfg_error) -> {test_server,my_apply,3}, {test_server,do_end_per_testcase,4}, {test_server,run_test_case_eval1,6}, - {test_server,run_test_case_eval,8}]}}}}}}, + {test_server,run_test_case_eval,9}]}}}}}}, {?eh,test_stats,{12,3,{0,18}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc14}}, {?eh,tc_done, {cfg_error_9_SUITE,tc14,{failed,{error,tc14_should_be_failed}}}}, - {?eh,test_stats,{12,4,{0,18}}}, - + {?eh,tc_start,{cfg_error_9_SUITE,tc15}}, + {?eh,tc_done, + {cfg_error_9_SUITE,tc15,{failed,{error,this_error_must_show}}}}, + {?eh,tc_start,{cfg_error_9_SUITE,tc16}}, + {?eh,tc_done, + {cfg_error_9_SUITE,tc16,{failed,{error,this_error_must_show}}}}, + {?eh,test_stats,{12,6,{0,18}}}, {?eh,tc_start,{cfg_error_9_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_9_SUITE,end_per_suite,ok}}, @@ -533,7 +561,7 @@ test_events(cfg_error) -> {?eh,tc_auto_skip,{cfg_error_10_SUITE,tc1, {failed,{cfg_error_10_SUITE,init_per_suite, {failed,fail_init_per_suite}}}}}, - {?eh,test_stats,{12,4,{0,19}}}, + {?eh,test_stats,{12,6,{0,19}}}, {?eh,tc_auto_skip,{cfg_error_10_SUITE,end_per_suite, {failed,{cfg_error_10_SUITE,init_per_suite, {failed,fail_init_per_suite}}}}}, @@ -542,40 +570,40 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, {?eh,tc_done,{cfg_error_11_SUITE,tc1, {skipped,{config_name_already_in_use,[dummy0]}}}}, - {?eh,test_stats,{12,4,{1,19}}}, + {?eh,test_stats,{12,6,{1,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, - {?eh,test_stats,{13,4,{1,19}}}, + {?eh,test_stats,{13,6,{1,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, {?eh,tc_done,{cfg_error_12_SUITE,tc1,{failed,{timetrap_timeout,500}}}}, - {?eh,test_stats,{13,5,{1,19}}}, + {?eh,test_stats,{13,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,5,{1,19}}}, + {?eh,test_stats,{14,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,5,{1,19}}}, + {?eh,test_stats,{15,7,{1,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,5,{1,19}}}, + {?eh,test_stats,{16,7,{1,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,5,{1,19}}}, + {?eh,test_stats,{17,7,{1,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,5,{1,19}}}, + {?eh,test_stats,{18,7,{1,19}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, @@ -603,7 +631,7 @@ test_events(lib_error) -> {?eh,test_stats,{0,2,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,lines_hang}}, {?eh,tc_done, - {lib_lines,do_hang,{failed,{timetrap_timeout,3000}}}}, + {lib_error_1_SUITE,lines_hang,{failed,{timetrap_timeout,3000}}}}, {?eh,test_stats,{0,3,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,lines_throw}}, {?eh,tc_done, @@ -752,7 +780,7 @@ test_events(timetrap_parallel) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,7}}, + {?eh,start_info,{1,1,8}}, {?eh,tc_done,{timetrap_3_SUITE,init_per_suite,ok}}, {parallel, [{?eh,tc_start, @@ -764,9 +792,12 @@ test_events(timetrap_parallel) -> {?eh,tc_start,{timetrap_3_SUITE,tc2}}, {?eh,tc_start,{timetrap_3_SUITE,tc3}}, {?eh,tc_start,{timetrap_3_SUITE,tc4}}, + {?eh,tc_start,{timetrap_3_SUITE,tc5}}, {?eh,tc_start,{timetrap_3_SUITE,tc6}}, {?eh,tc_start,{timetrap_3_SUITE,tc7}}, {?eh,tc_done, + {timetrap_3_SUITE,tc5,ok}}, + {?eh,tc_done, {timetrap_3_SUITE,tc1,{failed,{timetrap_timeout,500}}}}, {?eh,tc_done, {timetrap_3_SUITE,tc2,{failed,{timetrap_timeout,1000}}}}, @@ -780,11 +811,90 @@ test_events(timetrap_parallel) -> {timetrap_3_SUITE,tc4,{failed,{timetrap_timeout,2000}}}}, {?eh,tc_done, {timetrap_3_SUITE,tc3,{failed,{timetrap_timeout,3000}}}}, - {?eh,test_stats,{0,7,{0,0}}}, + {?eh,test_stats,{1,7,{0,0}}}, {?eh,tc_start, {timetrap_3_SUITE,{end_per_group,g1,[parallel]}}}, {?eh,tc_done, {timetrap_3_SUITE,{end_per_group,g1,[parallel]},ok}}]}, {?eh,tc_done,{timetrap_3_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, - {?eh,stop_logging,[]}]. + {?eh,stop_logging,[]}]; + +test_events(timetrap_fun) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,start_info,{4,4,17}}, + {?eh,tc_done,{timetrap_4_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_4_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc0,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_4_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc1,{failed,{timetrap_timeout,2000}}}}, + {?eh,tc_start,{timetrap_4_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc2,{failed,{timetrap_timeout,500}}}}, + {?eh,tc_start,{timetrap_4_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_4_SUITE,tc3,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_done,{timetrap_4_SUITE,end_per_suite,ok}}, + + {?eh,tc_done,{timetrap_5_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_5_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc0,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc1,{skipped,{timetrap_error,kaboom}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc2,{skipped,{timetrap_error,kaboom}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc3, + {skipped,{invalid_time_format,{timetrap_utils,timetrap_val,[5000]}}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc4}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc4,{skipped,{invalid_time_format,'_'}}}}, + {?eh,test_stats,{0,5,{0,4}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc5}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc5,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc6}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc6,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_5_SUITE,tc7}}, + {?eh,tc_done, + {timetrap_5_SUITE,tc7,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,8,{0,4}}}, + {?eh,tc_done,{timetrap_5_SUITE,end_per_suite,ok}}, + + {?eh,tc_start,{timetrap_6_SUITE,init_per_suite}}, + {?eh,tc_done, + {timetrap_6_SUITE,init_per_suite,{skipped,{timetrap_error,kaboom}}}}, + {?eh,tc_auto_skip, + {timetrap_6_SUITE,tc0,{fw_auto_skip,{timetrap_error,kaboom}}}}, + {?eh,test_stats,{0,8,{0,5}}}, + {?eh,tc_auto_skip, + {timetrap_6_SUITE,end_per_suite,{fw_auto_skip,{timetrap_error,kaboom}}}}, + + {?eh,tc_done,{timetrap_7_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{timetrap_7_SUITE,tc0}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc0,{failed,{timetrap_timeout,1000}}}}, + {?eh,tc_start,{timetrap_7_SUITE,tc1}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc1,{failed,{timetrap_timeout,2000}}}}, + {?eh,tc_start,{timetrap_7_SUITE,tc2}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc2,{failed,{timetrap_timeout,500}}}}, + {?eh,tc_start,{timetrap_7_SUITE,tc3}}, + {?eh,tc_done, + {timetrap_7_SUITE,tc3,{failed,{timetrap_timeout,1000}}}}, + {?eh,test_stats,{0,12,{0,5}}}, + {?eh,tc_done,{timetrap_7_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl index c4e0d72948..f292985c0c 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/cfg_error_9_SUITE.erl @@ -76,7 +76,7 @@ init_per_testcase(tc1, Config) -> Config; init_per_testcase(tc2, Config) -> ct:comment("init_per_testcase(tc2) timeout"), - timer:sleep(5000), + ct:sleep(5000), Config; init_per_testcase(tc3, Config) -> badmatch = ?config(void, Config), @@ -96,22 +96,20 @@ init_per_testcase(_, Config) -> %%-------------------------------------------------------------------- end_per_testcase(tc11, _Config) -> ct:comment("A warning should be printed"), - exit(warning_should_be_printed), - done; + exit(warning_should_be_printed); end_per_testcase(tc12, _Config) -> ct:comment("A warning should be printed"), - timer:sleep(5000), - done; + ct:sleep(5000); end_per_testcase(tc13, Config) -> ct:comment("A warning should be printed"), - badmatch = ?config(void, Config), - done; + badmatch = ?config(void, Config); end_per_testcase(tc14, Config) -> ok = ?config(tc_status, Config), {fail,tc14_should_be_failed}; end_per_testcase(tc15, Config) -> - {failed,byebye} = ?config(tc_status, Config), - ok; + exit(kaboom); +end_per_testcase(tc16, Config) -> + ct:sleep(5000); end_per_testcase(_TestCase, _Config) -> done. @@ -139,7 +137,7 @@ groups() -> %%-------------------------------------------------------------------- all() -> [tc1,tc2,tc3,tc4,tc5,tc6,tc7, - tc11,tc12,tc13,tc14]. + tc11,tc12,tc13,tc14,tc15,tc16]. tc1(_) -> fini. @@ -189,4 +187,6 @@ tc14(_) -> ct:comment("This one should be failed by eptc"), yes. tc15(_) -> - exit(byebye). + exit(this_error_must_show). +tc16(_) -> + exit(this_error_must_show). diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_3_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_3_SUITE.erl new file mode 100644 index 0000000000..8271b23afe --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_3_SUITE.erl @@ -0,0 +1,146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_3_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 3). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,?TO}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{g1,[parallel],[tc0,tc1,tc2,tc3,tc4,tc5,tc6,tc7]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [{group,g1}]. + +tc0() -> + [{timetrap,2000}]. +tc0(_) -> + ct:comment("TO after 2 sec"), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,500}]. +tc1(_) -> + ct:comment("TO after 1/2 sec"), + ct:sleep({seconds,5}), + ok. + +tc2() -> + [{timetrap,1000}]. +tc2(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,5}), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc4() -> + [{timetrap,2000}]. +tc4(_) -> + ct:comment(io_lib:format("TO after 2 sec", [])), + ct:sleep({seconds,5}), + ok. + +tc5() -> + [{timetrap,2000}]. +tc5(_) -> + ct:comment("No timeout"), + ct:sleep({seconds,1}), + ok. + +tc6() -> + [{timetrap,1000}]. +tc6(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,5}), + ok. + +tc7() -> + [{timetrap,1500}]. +tc7(_) -> + ct:comment("TO after 1 1/2 sec"), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_4_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_4_SUITE.erl new file mode 100644 index 0000000000..d902454f09 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_4_SUITE.erl @@ -0,0 +1,135 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_4_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2,tc3]. + +tc0(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[2000]}}]. +tc1(_) -> + ct:comment("TO after 2 sec"), + ct:sleep({seconds,5}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep(1000), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_5_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_5_SUITE.erl new file mode 100644 index 0000000000..c5d4b5062e --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_5_SUITE.erl @@ -0,0 +1,155 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_5_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap, fun() -> timetrap_utils:timetrap_val({seconds,?TO}) end}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2,tc3,tc4,tc5,tc6,tc7]. + +tc0(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]. +tc1(_) -> + exit(this_should_not_execute). + +tc2() -> + [{timetrap,fun() -> exit(kaboom) end}]. +tc2(_) -> + exit(this_should_not_execute). + +tc3() -> + [{timetrap,{timetrap_utils,timetrap_err_mfa,[]}}]. +tc3(_) -> + exit(this_should_not_execute). + +tc4() -> + [{timetrap,fun() -> timetrap_utils:timetrap_err_fun() end}]. +tc4(_) -> + exit(this_should_not_execute). + +tc5() -> + [{timetrap,{timetrap_utils,timetrap_timeout,[{seconds,40}, + {seconds,1}]}}]. +tc5(_) -> + ct:comment("TO after 40+1 sec"), + ct:sleep({seconds,42}), + ok. + +tc6() -> + [{timetrap,fun() -> ct:sleep(6000), 1000 end}]. +tc6(_) -> + ct:comment("TO after 6+1 sec"), + ct:sleep({seconds,10}). + +tc7(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_6_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_6_SUITE.erl new file mode 100644 index 0000000000..90467ff752 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_6_SUITE.erl @@ -0,0 +1,114 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_6_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap, fun() -> exit(kaboom) end}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0]. + +tc0(_) -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_7_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_7_SUITE.erl new file mode 100644 index 0000000000..b25b7770a7 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_7_SUITE.erl @@ -0,0 +1,137 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_7_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 1). +-define(HANG, 6). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_timeout,[{seconds,?HANG}, + {seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + []. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [tc0,tc1,tc2,tc3]. + +tc0(_) -> + ct:comment(io_lib:format("TO after ~w+~w sec", [?HANG,?TO])), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[2000]}}]. +tc1(_) -> + ct:comment("TO after 2 sec"), + ct:sleep({seconds,5}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep(1000), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w+~w sec", [?HANG,?TO])), + ct:sleep({seconds,5}), + ok. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_utils.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_utils.erl new file mode 100644 index 0000000000..fcde6cd701 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_utils.erl @@ -0,0 +1,43 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(timetrap_utils). + +-export([timetrap_val/1, + timetrap_err_fun/0, + timetrap_err_mfa/0, + timetrap_exit/1, + timetrap_timeout/2]). + +timetrap_val(Val) -> + Val. + +timetrap_err_fun() -> + fun() -> 5000 end. + +timetrap_err_mfa() -> + {?MODULE,timetrap_val,[5000]}. + +timetrap_exit(Reason) -> + exit(Reason). + +timetrap_timeout(Sleep, Val) -> + ct:sleep(Sleep), + Val. + diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index f33be8a9d4..940d791b15 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -173,16 +173,14 @@ test_events(missing_conf) -> {?eh,start_info,{1,1,2}}, {?eh,tc_start,{ct_framework,ct_init_per_group}}, {?eh,tc_done,{ct_framework,ct_init_per_group,ok}}, - {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc1}}, {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, - {?eh,test_stats,{2,0,{0,0}}}, + {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc2}}, {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, - {?eh,test_stats,{3,0,{0,0}}}, + {?eh,test_stats,{2,0,{0,0}}}, {?eh,tc_start,{ct_framework,ct_end_per_group}}, {?eh,tc_done,{ct_framework,ct_end_per_group,ok}}, - {?eh,test_stats,{4,0,{0,0}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl index 47cea190dd..bda7d91161 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_1_SUITE.erl @@ -143,7 +143,7 @@ tc1(_) -> exit(should_have_been_skipped). tc2(_) -> - exit(should_have_been_skipped). + timeout_in_end_per_testcase. tc3(_) -> timer:sleep(5000). diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index e85e8e6ad3..6df02d12b7 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -23,7 +23,6 @@ %%% -module(ct_test_support). --include_lib("test_server/include/test_server.hrl"). -include_lib("common_test/include/ct_event.hrl"). -include_lib("common_test/include/ct.hrl"). diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index f77629b4d1..4782a32933 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1,3 +1,3 @@ -COMMON_TEST_VSN = 1.5.4 +COMMON_TEST_VSN = 1.5.5 diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 2cb33a9a69..49f97686a0 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -612,6 +612,7 @@ do_run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> print(minor, "Current directory is ~p\n", [Cwd]), print_timestamp(minor,"Started at "), TCCallback = get(test_server_testcase_callback), + LogOpts = get(test_server_logopts), Ref = make_ref(), OldGLeader = group_leader(), %% Set ourself to group leader for the spawned process @@ -621,7 +622,7 @@ do_run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> fun() -> run_test_case_eval(Mod, Func, Args, Name, Ref, RunInit, TimetrapData, - TCCallback) + LogOpts, TCCallback) end), group_leader(OldGLeader, self()), put(test_server_detected_fail, []), @@ -733,15 +734,23 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> print(Detail,Format,Args), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); {comment,NewComment} -> + NewComment1 = test_server_ctrl:to_string(NewComment), + NewComment2 = test_server_sup:framework_call(format_comment, + [NewComment1], + NewComment1), Terminate1 = case Terminate of {true,{Time,Value,Loc,Opts,_OldComment}} -> - {true,{Time,Value,mod_loc(Loc),Opts,NewComment}}; + {true,{Time,Value,mod_loc(Loc),Opts,NewComment2}}; Other -> Other end, - run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate1,NewComment,CurrConf); - {set_curr_conf,NewCurrConf} -> + run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate1,NewComment2,CurrConf); + {read_comment,From} -> + From ! {self(),read_comment,Comment}, + run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); + {set_curr_conf,From,NewCurrConf} -> + From ! {self(),set_curr_conf,ok}, run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,NewCurrConf); {'EXIT',Pid,{Ref,Time,Value,Loc,Opts}} -> RetVal = {Time/1000000,Value,mod_loc(Loc),Opts,Comment}, @@ -753,7 +762,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case mod_loc(Loc) of {FwMod,FwFunc,framework} -> %% timout during framework call - spawn_fw_call(FwMod,FwFunc,Pid, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid, {framework_error,{timetrap,TVal}}, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate, @@ -779,7 +788,8 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> %% group leader process or io will cause deadlock, %% so we spawn a dedicated process for the operation %% and let the group leader go back to handle io. - spawn_fw_call(Mod,Func,Pid,{timetrap_timeout,TVal}, + spawn_fw_call(Mod,Func,CurrConf,Pid, + {timetrap_timeout,TVal}, Loc1,self(),Comment), undefined end, @@ -790,12 +800,13 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case mod_loc(Loc) of {FwMod,FwFunc,framework} -> %% timout during framework call - spawn_fw_call(FwMod,FwFunc,Pid, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid, {framework_error,{timetrap,TVal}}, unknown,self(),Comment); Loc1 -> {Mod,_Func} = get_mf(Loc1), - spawn_fw_call(Mod,InitOrEnd,Pid,{timetrap_timeout,TVal}, + spawn_fw_call(Mod,InitOrEnd,CurrConf,Pid, + {timetrap_timeout,TVal}, Loc1,self(),Comment) end, run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); @@ -804,7 +815,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case mod_loc(AbortLoc) of {FwMod,FwFunc,framework} -> %% abort during framework call - spawn_fw_call(FwMod,FwFunc,Pid, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid, {framework_error,ErrorMsg}, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate, @@ -828,7 +839,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> {EndConfPid,{Mod,Func},Conf}; _ -> {Mod,Func} = get_mf(Loc1), - spawn_fw_call(Mod,Func,Pid,ErrorMsg, + spawn_fw_call(Mod,Func,CurrConf,Pid,ErrorMsg, Loc1,self(),Comment), undefined end, @@ -839,17 +850,18 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> %% result of an exit(TestCase,kill) call, which is the %% only way to abort a testcase process that traps exits %% (see abort_current_testcase) - spawn_fw_call(undefined,undefined,Pid,testcase_aborted_or_killed, + spawn_fw_call(undefined,undefined,CurrConf,Pid, + testcase_aborted_or_killed, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); {fw_error,{FwMod,FwFunc,FwError}} -> - spawn_fw_call(FwMod,FwFunc,Pid,{framework_error,FwError}, + spawn_fw_call(FwMod,FwFunc,CurrConf,Pid,{framework_error,FwError}, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf); _Other -> %% the testcase has terminated because of Reason (e.g. an exit %% because a linked process failed) - spawn_fw_call(undefined,undefined,Pid,Reason, + spawn_fw_call(undefined,undefined,CurrConf,Pid,Reason, unknown,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf) end; @@ -857,7 +869,7 @@ run_test_case_msgloop(Ref, Pid, CaptureStdout, Terminate, Comment, CurrConf) -> case CurrConf of {EndConfPid,{Mod,Func},_Conf} -> {_Mod,_Func,TCPid,TCExitReason,Loc} = Data, - spawn_fw_call(Mod,Func,TCPid,TCExitReason,Loc,self(),Comment), + spawn_fw_call(Mod,Func,CurrConf,TCPid,TCExitReason,Loc,self(),Comment), run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,undefined); _ -> run_test_case_msgloop(Ref,Pid,CaptureStdout,Terminate,Comment,CurrConf) @@ -928,7 +940,7 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) -> ok end, Supervisor ! {self(),end_conf} - end, + end, Pid = spawn_link(EndConfApply), receive {Pid,end_conf} -> @@ -941,50 +953,72 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) -> end, spawn_link(EndConfProc). -spawn_fw_call(Mod,{init_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why, +spawn_fw_call(Mod,{init_per_testcase,Func},_,Pid,{timetrap_timeout,TVal}=Why, Loc,SendTo,Comment) -> FwCall = fun() -> - Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, - %% if init_per_testcase fails, the test case - %% should be skipped - case catch do_end_tc_call(Mod,Func,{Pid,Skip,[[]]},Why) of - {'EXIT',FwEndTCErr} -> - exit({fw_notify_done,end_tc,FwEndTCErr}); - _ -> - ok - end, - %% finished, report back - SendTo ! {self(),fw_notify_done, - {TVal/1000,Skip,Loc,[],Comment}} + Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, + %% if init_per_testcase fails, the test case + %% should be skipped + case catch do_end_tc_call(Mod,Func, Loc, {Pid,Skip,[[]]}, Why) of + {'EXIT',FwEndTCErr} -> + exit({fw_notify_done,end_tc,FwEndTCErr}); + _ -> + ok + end, + %% finished, report back + SendTo ! {self(),fw_notify_done, + {TVal/1000,Skip,Loc,[],Comment}} end, spawn_link(FwCall); -spawn_fw_call(Mod,{end_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why, - Loc,SendTo,_Comment) -> +spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid, + {timetrap_timeout,TVal}=Why,_Loc,SendTo,Comment) -> + %%! This is a temporary fix that keeps Test Server alive during + %%! execution of a parallel test case group, when sometimes + %%! this clause gets called with EndConf == undefined. See OTP-9594 + %%! for more info. + EndConf1 = if EndConf == undefined -> + [{tc_status,{failed,{Mod,end_per_testcase,Why}}}]; + true -> + EndConf + end, FwCall = fun() -> - Conf = [{tc_status,ok}], - %% if end_per_testcase fails, the test case should be - %% reported successful with a warning printed as comment - case catch do_end_tc_call(Mod,Func,{Pid, - {failed,{Mod,end_per_testcase,Why}}, - [Conf]}, Why) of - {'EXIT',FwEndTCErr} -> - exit({fw_notify_done,end_tc,FwEndTCErr}); - _ -> - ok - end, - %% finished, report back - SendTo ! {self(),fw_notify_done, - {TVal/1000,{error,{Mod,end_per_testcase,Why}},Loc,[], - ["<font color=\"red\">" - "WARNING: end_per_testcase timed out!" - "</font>"]}} + {RetVal,Report} = + case proplists:get_value(tc_status, EndConf1) of + undefined -> + E = {failed,{Mod,end_per_testcase,Why}}, + {E,E}; + E = {failed,Reason} -> + {E,{error,Reason}}; + Result -> + E = {failed,{Mod,end_per_testcase,Why}}, + {Result,E} + end, + FailLoc = proplists:get_value(tc_fail_loc, EndConf1), + case catch do_end_tc_call(Mod,Func, FailLoc, + {Pid,Report,[EndConf1]}, Why) of + {'EXIT',FwEndTCErr} -> + exit({fw_notify_done,end_tc,FwEndTCErr}); + _ -> + ok + end, + %% if end_per_testcase fails a warning should be + %% printed as comment + Comment1 = if Comment == "" -> ""; + true -> Comment ++ "<br>" + end, + %% finished, report back + SendTo ! {self(),fw_notify_done, + {TVal/1000,RetVal,FailLoc,[], + [Comment1,"<font color=\"red\">" + "WARNING: end_per_testcase timed out!" + "</font>"]}} end, spawn_link(FwCall); -spawn_fw_call(FwMod,FwFunc,_Pid,{framework_error,FwError},_,SendTo,_Comment) -> +spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo,_Comment) -> FwCall = fun() -> test_server_sup:framework_call(report, [framework_error, @@ -999,7 +1033,7 @@ spawn_fw_call(FwMod,FwFunc,_Pid,{framework_error,FwError},_,SendTo,_Comment) -> end, spawn_link(FwCall); -spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) -> +spawn_fw_call(Mod,Func,_,Pid,Error,Loc,SendTo,Comment) -> FwCall = fun() -> case catch fw_error_notify(Mod,Func,[], @@ -1011,7 +1045,8 @@ spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) -> ok end, Conf = [{tc_status,{failed,timetrap_timeout}}], - case catch do_end_tc_call(Mod,Func,{Pid,Error,[Conf]},Error) of + case catch do_end_tc_call(Mod,Func, Loc, + {Pid,Error,[Conf]},Error) of {'EXIT',FwEndTCErr} -> exit({fw_notify_done,end_tc,FwEndTCErr}); _ -> @@ -1072,8 +1107,9 @@ job_proxy_msgloop() -> %% or sends a message {failed, File, Line} to it's group_leader run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, - TimetrapData, TCCallback) -> - put(test_server_multiply_timetraps,TimetrapData), + TimetrapData, LogOpts, TCCallback) -> + put(test_server_multiply_timetraps, TimetrapData), + put(test_server_logopts, LogOpts), {{Time,Value},Loc,Opts} = case test_server_sup:framework_call(init_tc,[?pl2a(Mod),Func,Args0], @@ -1081,22 +1117,26 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, {ok,Args} -> run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback); Error = {error,_Reason} -> - NewResult = do_end_tc_call(Mod,Func,{Error,Args0}, + Where = {Mod,Func}, + NewResult = do_end_tc_call(Mod,Func, Where, {Error,Args0}, {skip,{failed,Error}}), - {{0,NewResult},{Mod,Func},[]}; + {{0,NewResult},Where,[]}; {fail,Reason} -> Conf = [{tc_status,{failed,Reason}} | hd(Args0)], + Where = {Mod,Func}, fw_error_notify(Mod, Func, Conf, Reason), - NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf]}, + NewResult = do_end_tc_call(Mod,Func, Where, {{error,Reason},[Conf]}, {fail,Reason}), - {{0,NewResult},{Mod,Func},[]}; + {{0,NewResult},Where,[]}; Skip = {skip,_Reason} -> - NewResult = do_end_tc_call(Mod,Func,{Skip,Args0},Skip), - {{0,NewResult},{Mod,Func},[]}; + Where = {Mod,Func}, + NewResult = do_end_tc_call(Mod,Func, Where, {Skip,Args0}, Skip), + {{0,NewResult},Where,[]}; {auto_skip,Reason} -> - NewResult = do_end_tc_call(Mod, Func, {{skip,Reason},Args0}, + Where = {Mod,Func}, + NewResult = do_end_tc_call(Mod,Func, Where, {{skip,Reason},Args0}, {skip,{fw_auto_skip,Reason}}), - {{0,NewResult},{Mod,Func},[]} + {{0,NewResult},Where,[]} end, exit({Ref,Time,Value,Loc,Opts}). @@ -1110,18 +1150,19 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> Skip = {skip,Reason} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}}], - NewRes = do_end_tc_call(Mod,Func,{Skip,[Conf]}, Skip), + NewRes = do_end_tc_call(Mod,Func, Line, {Skip,[Conf]}, Skip), {{0,NewRes},Line,[]}; {skip_and_save,Reason,SaveCfg} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}], - NewRes = do_end_tc_call(Mod, Func, {{skip,Reason},[Conf]}, - {skip, Reason}), + NewRes = do_end_tc_call(Mod,Func, Line, {{skip,Reason},[Conf]}, + {skip,Reason}), {{0,NewRes},Line,[]}; FailTC = {fail,Reason} -> % user fails the testcase EndConf = [{tc_status,{failed,Reason}} | hd(Args)], fw_error_notify(Mod, Func, EndConf, Reason), - NewRes = do_end_tc_call(Mod, Func, {{error,Reason},[EndConf]}, + NewRes = do_end_tc_call(Mod,Func, {Mod,Func}, + {{error,Reason},[EndConf]}, FailTC), {{0,NewRes},{Mod,Func},[]}; {ok,NewConf} -> @@ -1129,47 +1170,61 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> %% call user callback function if defined NewConf1 = user_callback(TCCallback, Mod, Func, init, NewConf), %% save current state in controller loop - group_leader() ! {set_curr_conf,{{Mod,Func},NewConf1}}, + sync_send(group_leader(),set_curr_conf,{{Mod,Func},NewConf1}, + 5000, fun() -> exit(no_answer_from_group_leader) end), put(test_server_loc, {Mod,Func}), %% execute the test case {{T,Return},Loc} = {ts_tc(Mod, Func, [NewConf1]),get_loc()}, {EndConf,TSReturn,FWReturn} = case Return of {E,TCError} when E=='EXIT' ; E==failed -> + ModLoc = mod_loc(Loc), fw_error_notify(Mod, Func, NewConf1, - TCError, mod_loc(Loc)), - {[{tc_status,{failed,TCError}}|NewConf1], + TCError, ModLoc), + {[{tc_status,{failed,TCError}}, + {tc_fail_loc,ModLoc}|NewConf1], Return,{error,TCError}}; SaveCfg={save_config,_} -> {[{tc_status,ok},SaveCfg|NewConf1],Return,ok}; {skip_and_save,Why,SaveCfg} -> Skip = {skip,Why}, - {[{tc_status,{skipped,Why}},{save_config,SaveCfg}|NewConf1], + {[{tc_status,{skipped,Why}}, + {save_config,SaveCfg}|NewConf1], Skip,Skip}; {skip,Why} -> {[{tc_status,{skipped,Why}}|NewConf1],Return,Return}; _ -> {[{tc_status,ok}|NewConf1],Return,ok} end, - %% clear current state in controller loop - group_leader() ! {set_curr_conf,undefined}, %% call user callback function if defined EndConf1 = user_callback(TCCallback, Mod, Func, 'end', EndConf), + %% update current state in controller loop + sync_send(group_leader(),set_curr_conf,EndConf1, + 5000, fun() -> exit(no_answer_from_group_leader) end), {FWReturn1,TSReturn1,EndConf2} = case end_per_testcase(Mod, Func, EndConf1) of SaveCfg1={save_config,_} -> {FWReturn,TSReturn,[SaveCfg1|lists:keydelete(save_config,1, EndConf1)]}; - {fail,ReasonToFail} -> % user has failed the testcase + {fail,ReasonToFail} -> + %% user has failed the testcase fw_error_notify(Mod, Func, EndConf1, ReasonToFail), {{error,ReasonToFail},{failed,ReasonToFail},EndConf1}; - {failed,{_,end_per_testcase,_}} = Failure -> % unexpected termination + {failed,{_,end_per_testcase,_}} = Failure when FWReturn == ok -> + %% unexpected termination in end_per_testcase + %% report this as the result to the framework {Failure,TSReturn,EndConf1}; _ -> + %% test case result should be reported to framework + %% no matter the status of end_per_testcase {FWReturn,TSReturn,EndConf1} end, + %% clear current state in controller loop + sync_send(group_leader(),set_curr_conf,undefined, + 5000, fun() -> exit(no_answer_from_group_leader) end), put(test_server_init_or_end_conf,undefined), - case do_end_tc_call(Mod, Func, {FWReturn1,[EndConf2]}, TSReturn1) of + case do_end_tc_call(Mod,Func, Loc, + {FWReturn1,[EndConf2]}, TSReturn1) of {failed,Reason} = NewReturn -> fw_error_notify(Mod,Func,EndConf2, Reason), {{T,NewReturn},{Mod,Func},[]}; @@ -1193,18 +1248,43 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> %% call user callback function if defined Return1 = user_callback(TCCallback, Mod, Func, 'end', Return), {Return2,Opts} = process_return_val([Return1], Mod, Func, - Args1, Loc, Return1), + Args1, {Mod,Func}, Return1), {{T,Return2},Loc,Opts} end. -do_end_tc_call(M,F,Res,Return) -> +do_end_tc_call(M,F, Loc, Res, Return) -> + FwMod = os:getenv("TEST_SERVER_FRAMEWORK"), + {Mod,Func} = + if FwMod == M ; FwMod == "undefined"; FwMod == false -> + {M,F}; + is_list(Loc) and (length(Loc)>1) -> + %% If failure in other module (M) than suite, try locate + %% suite name in Loc list and call end_tc with Suite:TestCase + %% instead of M:F. + GetSuite = fun(S,TC) -> + case lists:reverse(atom_to_list(S)) of + [$E,$T,$I,$U,$S,$_|_] -> [{S,TC}]; + _ -> [] + end + end, + case lists:flatmap(fun({S,TC,_}) -> GetSuite(S,TC); + ({{S,TC},_}) -> GetSuite(S,TC); + ({S,TC}) -> GetSuite(S,TC); + (_) -> [] + end, Loc) of + [] -> + {M,F}; + [FoundSuite|_] -> + FoundSuite + end; + true -> + {M,F} + end, + Ref = make_ref(), - case os:getenv("TEST_SERVER_FRAMEWORK") of - FW when FW == "ct_framework"; - FW == "undefined"; - FW == false -> + if FwMod == "ct_framework" ; FwMod == "undefined"; FwMod == false -> case test_server_sup:framework_call( - end_tc, [?pl2a(M),F,Res, Return], ok) of + end_tc, [?pl2a(Mod),Func,Res, Return], ok) of {fail,FWReason} -> {failed,FWReason}; ok -> @@ -1217,9 +1297,9 @@ do_end_tc_call(M,F,Res,Return) -> NewReturn -> NewReturn end; - Other -> - case test_server_sup:framework_call( - end_tc, [Other,F,Res], Ref) of + true -> + case test_server_sup:framework_call(FwMod, end_tc, + [?pl2a(Mod),Func,Res], Ref) of {fail,FWReason} -> {failed,FWReason}; _Else -> @@ -1242,7 +1322,7 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) -> true -> % must be return value from end conf case process_return_val1(Return, M,F,A, Loc, Final, []); false -> % must be Config value from init conf case - case do_end_tc_call(M,F,{ok,A}, Return) of + case do_end_tc_call(M, F, Loc, {ok,A}, Return) of {failed, FWReason} = Failed -> fw_error_notify(M,F,A, FWReason), {Failed, []}; @@ -1259,8 +1339,9 @@ process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts) when E=='EXIT'; E==failed -> fw_error_notify(M,F,A, TCError, mod_loc(Loc)), - case do_end_tc_call(M,F,{{error,TCError}, - [[{tc_status,{failed,TCError}}|Args]]}, Failed) of + case do_end_tc_call(M,F, Loc, {{error,TCError}, + [[{tc_status,{failed,TCError}}|Args]]}, + Failed) of {failed,FWReason} -> {{failed,FWReason},SaveOpts}; NewReturn -> @@ -1277,8 +1358,8 @@ process_return_val1([RetVal={Tag,_}|Opts], M,F,A, Loc, _, SaveOpts) when Tag==sk process_return_val1(Opts, M,F,A, Loc, RetVal, SaveOpts); process_return_val1([_|Opts], M,F,A, Loc, Final, SaveOpts) -> process_return_val1(Opts, M,F,A, Loc, Final, SaveOpts); -process_return_val1([], M,F,A, _Loc, Final, SaveOpts) -> - case do_end_tc_call(M,F,{Final,A}, Final) of +process_return_val1([], M,F,A, Loc, Final, SaveOpts) -> + case do_end_tc_call(M,F, Loc, {Final,A}, Final) of {failed,FWReason} -> {{failed,FWReason},SaveOpts}; NewReturn -> @@ -1389,10 +1470,14 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> ok catch throw:Other -> + Comment0 = case read_comment() of + "" -> ""; + Cmt -> Cmt ++ "<br>" + end, set_loc(erlang:get_stacktrace()), - comment(io_lib:format("<font color=\"red\">" + comment(io_lib:format("~s<font color=\"red\">" "WARNING: ~w thrown!" - "</font>\n",[EndFunc])), + "</font>\n",[Comment0,EndFunc])), group_leader() ! {printout,12, "WARNING: ~w thrown!\n" "Reason: ~p\n" @@ -1408,9 +1493,13 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> exit -> {'EXIT',Reason}; error -> {'EXIT',{Reason,Stk}} end, - comment(io_lib:format("<font color=\"red\">" + Comment0 = case read_comment() of + "" -> ""; + Cmt -> Cmt ++ "<br>" + end, + comment(io_lib:format("~s<font color=\"red\">" "WARNING: ~w crashed!" - "</font>\n",[EndFunc])), + "</font>\n",[Comment0,EndFunc])), group_leader() ! {printout,12, "WARNING: ~w crashed!\n" "Reason: ~p\n" @@ -1900,11 +1989,54 @@ time_ms({seconds,N}) -> seconds(N); time_ms({Other,_N}) -> format("=== ERROR: Invalid time specification: ~p. " "Should be seconds, minutes, or hours.~n", [Other]), - exit({invalid_time_spec,Other}); + exit({invalid_time_format,Other}); time_ms(Ms) when is_integer(Ms) -> Ms; time_ms(infinity) -> infinity; -time_ms(Other) -> exit({invalid_time_spec,Other}). +time_ms(Fun) when is_function(Fun) -> + time_ms_apply(Fun); +time_ms({M,F,A}=MFA) when is_atom(M), is_atom(F), is_list(A) -> + time_ms_apply(MFA); +time_ms(Other) -> exit({invalid_time_format,Other}). + +time_ms_apply(Func) -> + time_ms_apply(Func, [5000,30000,60000,infinity]). + +time_ms_apply(Func, TOs) -> + Apply = fun() -> + case Func of + {M,F,A} -> + exit({self(),apply(M, F, A)}); + Fun -> + exit({self(),Fun()}) + end + end, + Pid = spawn(Apply), + Ref = monitor(process, Pid), + time_ms_wait(Func, Pid, Ref, TOs). +time_ms_wait(Func, Pid, Ref, [TO|TOs]) -> + receive + {'DOWN',Ref,process,Pid,{Pid,Result}} -> + time_ms_check(Result); + {'DOWN',Ref,process,Pid,Error} -> + exit({timetrap_error,Error}) + after + TO -> + format("=== WARNING: No return from timetrap function ~p~n", [Func]), + time_ms_wait(Func, Pid, Ref, TOs) + end; +%% this clause will never execute if 'infinity' is in TOs list, that's ok! +time_ms_wait(Func, Pid, Ref, []) -> + demonitor(Ref), + exit(Pid, kill), + exit({timetrap_error,{no_return_from_timetrap_function,Func}}). + +time_ms_check(MFA = {M,F,A}) when is_atom(M), is_atom(F), is_list(A) -> + exit({invalid_time_format,MFA}); +time_ms_check(Fun) when is_function(Fun) -> + exit({invalid_time_format,Fun}); +time_ms_check(Other) -> + time_ms(Other). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timetrap_cancel(Handle) -> ok @@ -1952,6 +2084,19 @@ hours(N) -> trunc(N * 1000 * 60 * 60). minutes(N) -> trunc(N * 1000 * 60). seconds(N) -> trunc(N * 1000). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% sync_send(Pid,Tag,Msg,Timeout,DoAfter) -> Result +%% +sync_send(Pid,Tag,Msg,Timeout,DoAfter) -> + Pid ! {Tag,self(),Msg}, + receive + {Pid,Tag,Result} -> + Result + after Timeout -> + DoAfter() + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timecall(M,F,A) -> {Time,Val} %% Time = float() @@ -2338,6 +2483,21 @@ comment(String) -> group_leader() ! {comment,String}, ok. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% read_comment() -> string() +%% +%% Read the current comment string stored in +%% state during test case execution. +read_comment() -> + MsgLooper = group_leader(), + MsgLooper ! {read_comment,self()}, + receive + {MsgLooper,read_comment,Comment} -> + Comment + after + 5000 -> + "" + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% os_type() -> OsType diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index f3445b742b..4fad86d16d 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -173,7 +173,7 @@ %%% TEST_SERVER INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([output/2, print/2, print/3, print_timestamp/2]). -export([start_node/3, stop_node/1, wait_for_node/1, is_release_available/1]). --export([format/1, format/2, format/3]). +-export([format/1, format/2, format/3, to_string/1]). -export([get_target_info/0]). -export([get_hosts/0]). -export([get_target_os_type/0]). @@ -1297,6 +1297,7 @@ terminate(_Reason, State) -> end, kill_all_jobs(State#state.jobs), test_server_node:stop(State#state.target_info), + test_server_h:restore(), ok. kill_all_jobs([{_Name,JobPid}|Jobs]) -> @@ -1349,6 +1350,10 @@ init_tester(Mod, Func, Args, Dir, Name, {SumLev,MajLev,MinLev}, put(test_server_minor_level, MinLev), put(test_server_random_seed, proplists:get_value(random_seed, ExtraTools)), put(test_server_testcase_callback, TCCallback), + %% before first print, read and set logging options + LogOpts = test_server_sup:framework_call(get_logopts, [], []), + put(test_server_logopts, LogOpts), + put(test_server_log_nl, not lists:member(no_nl, LogOpts)), StartedExtraTools = start_extra_tools(ExtraTools), {TimeMy,Result} = ts_tc(Mod, Func, Args), put(test_server_common_io_handler, undefined), @@ -1664,6 +1669,11 @@ do_test_cases(TopCases, SkipCases, Config, TimetrapData) when is_list(TopCases), is_tuple(TimetrapData) -> start_log_file(), + FwMod = + case os:getenv("TEST_SERVER_FRAMEWORK") of + FW when FW =:= false; FW =:= "undefined" -> ?MODULE; + FW -> list_to_atom(FW) + end, case collect_all_cases(TopCases, SkipCases) of {error,Why} -> print(1, "Error starting: ~p", [Why]), @@ -1676,11 +1686,11 @@ do_test_cases(TopCases, SkipCases, put(test_server_cases, N), put(test_server_case_num, 0), TestSpec = - add_init_and_end_per_suite(TestSpec0, undefined, undefined), - + add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod), TI = get_target_info(), - print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]}, - {" (with repeated test cases)",[]})]), + print(1, "Starting test~s", + [print_if_known(N, {", ~w test cases",[N]}, + {" (with repeated test cases)",[]})]), Test = get(test_server_name), test_server_sup:framework_call(report, [tests_start,{Test,N}]), @@ -1709,13 +1719,12 @@ do_test_cases(TopCases, SkipCases, print(html, "<br>Used Erlang ~s in <tt>~s</tt>.\n", [erlang:system_info(version), code:root_dir()]), - case os:getenv("TEST_SERVER_FRAMEWORK") of - FW when FW =:= false; FW =:= "undefined" -> + if FwMod == ?MODULE -> print(html, "<p>Target:<br>\n"), print_who(TI#target_info.host, TI#target_info.username), print(html, "<br>Used Erlang ~s in <tt>~s</tt>.\n", [TI#target_info.version, TI#target_info.root_dir]); - _ -> + true -> case test_server_sup:framework_call(target_info, []) of TargetInfo when is_list(TargetInfo), length(TargetInfo) > 0 -> @@ -1884,11 +1893,12 @@ start_minor_log_file1(Mod, Func, LogDir, AbsName) -> []), SrcListing = downcase(cast_to_list(Mod)) ++ ?src_listing_ext, - case filelib:is_file(filename:join(LogDir, SrcListing)) of - true -> + case {filelib:is_file(filename:join(LogDir, SrcListing)), + lists:member(no_src, get(test_server_logopts))} of + {true,false} -> print(Lev, "<a href=\"~s#~s\">source code for ~p:~p/1</a>\n", [SrcListing,Func,Mod,Func]); - false -> ok + _ -> ok end, io:fwrite(Fd, "<pre>\n", []), @@ -2005,54 +2015,69 @@ copy_html_file(Src, DestDir) -> end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% add_init_and_end_per_suite(TestSpec, Mod, Ref) -> NewTestSpec +%% add_init_and_end_per_suite(TestSpec, Mod, Ref, FwMod) -> NewTestSpec %% %% Expands TestSpec with an initial init_per_suite, and a final %% end_per_suite element, per each discovered suite in the list. -add_init_and_end_per_suite([{make,_,_}=Case|Cases], LastMod, LastRef) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([{skip_case,{{Mod,all},_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> +add_init_and_end_per_suite([{make,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{{Mod,all},_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_end_per_suite_and_skip(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, + LastRef, FwMod) -> + %% if Mod == FwMod, this conf test is (probably) a test case group where + %% the init- and end-functions are missing in the suite, and if so, + %% the suite name should be stored as {suite,Suite} in Props + case proplists:get_value(suite, Props) of + Suite when Suite =/= undefined, Suite =/= LastMod -> + {PreCases, NextMod, NextRef} = + do_add_init_and_end_per_suite(LastMod, LastRef, Suite), + Case1 = {conf,Ref,proplists:delete(suite,Props),{FwMod,Func}}, + PreCases ++ [Case1|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; + _ -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)] + end; +add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, + LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) + when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([{Mod,_,_}=Case|Cases], LastMod, LastRef) - when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([{Mod,_,_}=Case|Cases], LastMod, LastRef, FwMod) + when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef)]; -add_init_and_end_per_suite([Case|Cases], LastMod, LastRef)-> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef)]; -add_init_and_end_per_suite([], _LastMod, undefined) -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; +add_init_and_end_per_suite([Case|Cases], LastMod, LastRef, FwMod)-> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([], _LastMod, undefined, _FwMod) -> []; -add_init_and_end_per_suite([], _LastMod, skipped_suite) -> +add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) -> []; -add_init_and_end_per_suite([], LastMod, LastRef) -> +add_init_and_end_per_suite([], LastMod, LastRef, _FwMod) -> [{conf,LastRef,[],{LastMod,end_per_suite}}]. do_add_init_and_end_per_suite(LastMod, LastRef, Mod) -> @@ -2101,7 +2126,12 @@ run_test_cases(TestSpec, Config, TimetrapData) -> maybe_open_job_sock(), - html_convert_modules(TestSpec, Config), + case lists:member(no_src, get(test_server_logopts)) of + true -> + ok; + false -> + html_convert_modules(TestSpec, Config) + end, run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []), @@ -2310,7 +2340,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], handle_test_case_io_and_status(), set_io_buffering(undefined), {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]), + test_server_sup:framework_call(report, [tc_auto_skip, + {?pl2a(Mod),Func,Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); _ -> @@ -2318,7 +2349,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], %% parallel group (io buffering is active) wait_for_cases(Ref), {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{?pl2a(Mod),Func,Comment}]), + test_server_sup:framework_call(report, [tc_auto_skip, + {?pl2a(Mod),Func,Comment}]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -3959,8 +3991,11 @@ progress(ok, _CaseNum, Mod, Func, _Loc, RetVal, Time, case RetVal of {comment,RetComment} -> String = to_string(RetComment), + HtmlCmt = test_server_sup:framework_call(format_comment, + [String], + String), print(major, "=result ok: ~s", [String]), - "<td>" ++ String ++ "</td>"; + "<td>" ++ HtmlCmt ++ "</td>"; _ -> print(major, "=result ok", []), case Comment0 of @@ -4345,14 +4380,18 @@ output_to_fd(Fd, [$=|Msg], internal) -> io:put_chars(Fd, [$=]), io:put_chars(Fd, Msg), io:put_chars(Fd, "\n"); + output_to_fd(Fd, Msg, internal) -> io:put_chars(Fd, [$=,$=,$=,$ ]), io:put_chars(Fd, Msg), io:put_chars(Fd, "\n"); + output_to_fd(Fd, Msg, _Sender) -> io:put_chars(Fd, Msg), - io:put_chars(Fd, "\n"). - + case get(test_server_log_nl) of + false -> ok; + _ -> io:put_chars(Fd, "\n") + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timestamp_filename_get(Leader) -> string() @@ -4665,7 +4704,7 @@ collect_case_invoke(Mod, Case, MFA, St) -> collect_subcases(Mod, Case, MFA, St, Suite) end; _ -> - Suite = test_server_sup:framework_call(get_suite, [?pl2a(Mod),Case],[]), + Suite = test_server_sup:framework_call(get_suite, [?pl2a(Mod),Case], []), collect_subcases(Mod, Case, MFA, St, Suite) end. diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl index ec9be52bd3..875f45eea6 100644 --- a/lib/test_server/src/test_server_sup.erl +++ b/lib/test_server/src/test_server_sup.erl @@ -26,7 +26,7 @@ cleanup_crash_dumps/0, crash_dump_dir/0, tar_crash_dumps/0, get_username/0, get_os_family/0, hostatom/0, hostatom/1, hoststr/0, hoststr/1, - framework_call/2,framework_call/3, + framework_call/2,framework_call/3,framework_call/4, format_loc/1, package_str/1, package_atom/1, call_trace/1]). -include("test_server_internal.hrl"). @@ -541,8 +541,9 @@ format_loc({Mod,Func}) when is_atom(Func) -> format_loc({Mod,Line}) when is_integer(Line) -> %% ?line macro is used ModStr = package_str(Mod), - case lists:reverse(ModStr) of - [$E,$T,$I,$U,$S,$_|_] -> + case {lists:member(no_src, get(test_server_logopts)), + lists:reverse(ModStr)} of + {false,[$E,$T,$I,$U,$S,$_|_]} -> io_lib:format("{~s,<a href=\"~s~s#~w\">~w</a>}", [ModStr,downcase(ModStr),?src_listing_ext, round_to_10(Line),Line]); @@ -558,8 +559,9 @@ format_loc1([{Mod,Func,Line}|Rest]) -> [" ",format_loc1({Mod,Func,Line}),",\n"|format_loc1(Rest)]; format_loc1({Mod,Func,Line}) -> ModStr = package_str(Mod), - case lists:reverse(ModStr) of - [$E,$T,$I,$U,$S,$_|_] -> + case {lists:member(no_src, get(test_server_logopts)), + lists:reverse(ModStr)} of + {false,[$E,$T,$I,$U,$S,$_|_]} -> io_lib:format("{~s,~w,<a href=\"~s~s#~w\">~w</a>}", [ModStr,Func,downcase(ModStr),?src_listing_ext, round_to_10(Line),Line]); diff --git a/lib/test_server/src/ts_erl_config.erl b/lib/test_server/src/ts_erl_config.erl index 640c8ddc9f..3b41f90d55 100644 --- a/lib/test_server/src/ts_erl_config.erl +++ b/lib/test_server/src/ts_erl_config.erl @@ -222,7 +222,6 @@ erl_interface(Vars,OsType) -> end, CrossCompile = case OsType of vxworks -> "true"; - ose -> "true"; _ -> "false" end, [{erl_interface_libpath, filename:nativename(LibPath)}, @@ -329,8 +328,6 @@ sock_libraries({win32, _}) -> sock_libraries({unix, _}) -> ""; % Included in general libraries if needed. sock_libraries(vxworks) -> - ""; -sock_libraries(ose) -> "". link_library(LibName,{win32, _}) -> @@ -339,8 +336,6 @@ link_library(LibName,{unix, _}) -> "lib" ++ LibName ++ ".a"; link_library(LibName,vxworks) -> "lib" ++ LibName ++ ".a"; -link_library(_LibName,ose) -> - ""; link_library(_LibName,_Other) -> exit({link_library, not_supported}). diff --git a/lib/test_server/test/test_server_SUITE.erl b/lib/test_server/test/test_server_SUITE.erl index 4c344717f0..a8532b08ab 100644 --- a/lib/test_server/test/test_server_SUITE.erl +++ b/lib/test_server/test/test_server_SUITE.erl @@ -119,6 +119,11 @@ test_server_conf02_SUITE(Config) -> run_test_server_tests(SuiteName, NCases, NFail, NExpected, NSucc, NUsrSkip, NAutoSkip, NActualSkip, NActualFail, NActualSucc, Config) -> + + ct:log("See test case log files under:~n~p~n", + [filename:join([proplists:get_value(priv_dir, Config), + SuiteName++".logs"])]), + Node = proplists:get_value(node, Config), {ok,_Pid} = rpc:call(Node,test_server_ctrl, start, []), rpc:call(Node, @@ -132,6 +137,7 @@ run_test_server_tests(SuiteName, NCases, NFail, NExpected, NSucc, end), rpc:call(Node,test_server_ctrl, stop, []), + {ok,#suite{ n_cases = NCases, n_cases_failed = NFail, n_cases_expected = NExpected, diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk index 1dd4a84ce9..563c1b6db6 100644 --- a/lib/test_server/vsn.mk +++ b/lib/test_server/vsn.mk @@ -1,2 +1,2 @@ -TEST_SERVER_VSN = 3.4.4 +TEST_SERVER_VSN = 3.4.5 |