aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/ct_run.erl
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2010-06-03 14:14:38 +0200
committerRaimo Niskanen <[email protected]>2010-06-09 16:19:20 +0200
commitc3a1e56608ebe08f1ddc07273d85ff9c2779de9b (patch)
tree3477d760c98c305a70076e45e74239c7d140c355 /lib/common_test/src/ct_run.erl
parent24f7909acb5d395756ce1912ab426090e369eb84 (diff)
downloadotp-c3a1e56608ebe08f1ddc07273d85ff9c2779de9b.tar.gz
otp-c3a1e56608ebe08f1ddc07273d85ff9c2779de9b.tar.bz2
otp-c3a1e56608ebe08f1ddc07273d85ff9c2779de9b.zip
Implement support for user controllable timetrap parameters (multiply and scale)
Documentation still missing.
Diffstat (limited to 'lib/common_test/src/ct_run.erl')
-rw-r--r--lib/common_test/src/ct_run.erl1226
1 files changed, 660 insertions, 566 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 021bda2951..96703031d2 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -24,7 +24,6 @@
-module(ct_run).
-
%% Script interface
-export([script_start/0,script_usage/0]).
@@ -46,11 +45,26 @@
-define(abs(Name), filename:absname(Name)).
-define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)).
+-record(opts, {vts,
+ shell,
+ cover,
+ coverspec,
+ step,
+ logdir,
+ config = [],
+ event_handlers = [],
+ include = [],
+ silent_connections,
+ multiply_timetraps = 1,
+ scale_timetraps = false,
+ testspecs = [],
+ tests}).
+
%%%-----------------------------------------------------------------
%%% @spec script_start() -> void()
%%%
%%% @doc Start tests via the run_test program or script.
-%%%
+%%%
%%% <p>Example:<br/><code>./run_test -config config.ctc -dir
%%% $TEST_DIR</code></p>
%%%
@@ -60,12 +74,18 @@
script_start() ->
process_flag(trap_exit, true),
Args = merge_arguments(init:get_arguments()),
+ case proplists:get_value(help, Args) of
+ undefined -> script_start(Args);
+ _ -> script_usage()
+ end.
+
+script_start(Args) ->
Tracing = start_trace(Args),
- Res =
+ Res =
case ct_repeat:loop_test(script, Args) of
- false ->
+ false ->
{ok,Cwd} = file:get_cwd(),
- CTVsn =
+ CTVsn =
case filename:basename(code:lib_dir(common_test)) of
CTBase when is_list(CTBase) ->
case string:tokens(CTBase, "-") of
@@ -76,7 +96,7 @@ script_start() ->
io:format("~nCommon Test~s starting (cwd is ~s)~n~n", [CTVsn,Cwd]),
Self = self(),
Pid = spawn_link(fun() -> script_start1(Self, Args) end),
- receive
+ receive
{'EXIT',Pid,Reason} ->
case Reason of
{user_error,What} ->
@@ -101,287 +121,285 @@ script_start() ->
Res.
script_start1(Parent, Args) ->
- VtsOrShell =
- case lists:keymember(vts, 1, Args) of
- true ->
- vts;
- false ->
- case lists:keymember(shell, 1, Args) of
- true -> shell;
- false -> false
- end
- end,
- LogDir =
- case lists:keysearch(logdir, 1, Args) of
- {value,{logdir,[LogD]}} -> LogD;
- false -> "."
- end,
- EvHandlers =
- case lists:keysearch(event_handler, 1, Args) of
- {value,{event_handler,Handlers}} ->
- lists:map(fun(H) -> {list_to_atom(H),[]} end, Handlers);
- false ->
- []
- end,
- Cover =
- case lists:keysearch(cover, 1, Args) of
- {value,{cover,CoverFile}} ->
- {cover,?abs(CoverFile)};
- false ->
- false
- end,
-
- case lists:keysearch(ct_decrypt_key, 1, Args) of
- {value,{_,[DecryptKey]}} ->
+ %% read general start flags
+ Vts = get_start_opt(vts, true, Args),
+ Shell = get_start_opt(shell, true, Args),
+ Cover = get_start_opt(cover, fun(CoverFile) -> ?abs(CoverFile) end, Args),
+ LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args),
+ MultTT = get_start_opt(multiply_timetraps, fun(MT) -> MT end, Args),
+ ScaleTT = get_start_opt(scale_timetraps, fun(CT) -> CT end, Args),
+ EvHandlers = get_start_opt(event_handler,
+ fun(Handlers) ->
+ lists:map(fun(H) ->
+ {list_to_atom(H),[]}
+ end, Handlers) end,
+ [], Args),
+
+ %% check flags and set corresponding application env variables
+
+ %% ct_decrypt_key | ct_decrypt_file
+ case proplists:get_value(ct_decrypt_key, Args) of
+ [DecryptKey] ->
application:set_env(common_test, decrypt, {key,DecryptKey});
- false ->
- case lists:keysearch(ct_decrypt_file, 1, Args) of
- {value,{_,[DecryptFile]}} ->
- application:set_env(common_test, decrypt,
+ undefined ->
+ case proplists:get_value(ct_decrypt_file, Args) of
+ [DecryptFile] ->
+ application:set_env(common_test, decrypt,
{file,filename:absname(DecryptFile)});
- false ->
+ undefined ->
application:unset_env(common_test, decrypt)
end
end,
-
- case lists:keysearch(no_auto_compile, 1, Args) of
- {value,_} ->
- application:set_env(common_test, auto_compile, false);
- false ->
- application:set_env(common_test, auto_compile, true),
-
- InclDirs =
- case lists:keysearch(include,1,Args) of
- {value,{include,Incl}} when is_list(hd(Incl)) ->
- Incl;
- {value,{include,Incl}} when is_list(Incl) ->
- [Incl];
+ %% no_auto_compile + include
+ IncludeDirs =
+ case proplists:get_value(no_auto_compile, Args) of
+ undefined ->
+ application:set_env(common_test, auto_compile, true),
+ InclDirs =
+ case proplists:get_value(include, Args) of
+ {include,Incl} when is_list(hd(Incl)) ->
+ Incl;
+ {include,Incl} when is_list(Incl) ->
+ [Incl];
+ undefined ->
+ []
+ end,
+ case os:getenv("CT_INCLUDE_PATH") of
false ->
- []
- end,
- case os:getenv("CT_INCLUDE_PATH") of
- false ->
- application:set_env(common_test, include, InclDirs);
- CtInclPath ->
- InclDirs1 = string:tokens(CtInclPath,[$:,$ ,$,]),
- application:set_env(common_test, include, InclDirs1++InclDirs)
- end
+ application:set_env(common_test, include, InclDirs),
+ InclDirs;
+ CtInclPath ->
+ AllInclDirs =
+ string:tokens(CtInclPath,[$:,$ ,$,]) ++ InclDirs,
+ application:set_env(common_test, include, AllInclDirs),
+ AllInclDirs
+ end;
+ _ ->
+ application:set_env(common_test, auto_compile, false),
+ []
+ end,
+ %% basic_html - used by ct_logs
+ case proplists:get_value(basic_html, Args) of
+ undefined ->
+ application:set_env(common_test, basic_html, false);
+ _ ->
+ application:set_env(common_test, basic_html, true)
end,
- case lists:keysearch(basic_html, 1, Args) of
- {value,_} ->
- application:set_env(common_test, basic_html, true);
- false ->
- application:set_env(common_test, basic_html, false)
- end,
+ StartOpts = #opts({vts = Vts, shell = Shell, cover = Cover,
+ logdir = LogDir, event_handlers = EvHandlers,
+ include = IncludeDirs,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT}),
- Result =
- case lists:keysearch(refresh_logs, 1, Args) of
- {value,{refresh_logs,Refresh}} ->
- LogDir1 = case Refresh of
- [] -> LogDir;
- [RefreshDir] -> ?abs(RefreshDir)
- end,
- {ok,Cwd} = file:get_cwd(),
- file:set_cwd(LogDir1),
- timer:sleep(500), % give the shell time to print version etc
- io:nl(),
- case catch ct_logs:make_all_suites_index(refresh) of
- {'EXIT',ASReason} ->
- file:set_cwd(Cwd),
- {error,{all_suites_index,ASReason}};
- _ ->
- case catch ct_logs:make_all_runs_index(refresh) of
- {'EXIT',ARReason} ->
- file:set_cwd(Cwd),
- {error,{all_runs_index,ARReason}};
- _ ->
- file:set_cwd(Cwd),
- io:format("Logs in ~s refreshed!~n~n", [LogDir1]),
- timer:sleep(500), % time to flush io before quitting
- ok
- end
- end;
- false ->
- Config = ct_config:prepare_config_list(Args),
- case Config of
- [] ->
- case install([{config,[]},
- {event_handler,EvHandlers}],
- LogDir) of
- ok ->
- script_start2(VtsOrShell, [], EvHandlers,
- Args, LogDir, Cover);
- Error ->
- Error
- end;
- Config ->
- case lists:keysearch(spec, 1, Args) of
- false ->
- case check_and_install_configfiles(Config,
- LogDir, EvHandlers) of
- ok ->
- script_start2(VtsOrShell, Config,
- EvHandlers, Args, LogDir,
- Cover);
- Error ->
- Error
- end;
- _ ->
- script_start2(VtsOrShell, Config,
- EvHandlers, Args, LogDir, Cover)
- end
- end
- end,
+ %% check if log files should be refreshed or go on to run tests...
+ Result = run_or_refresh(StartOpts, Args),
+ %% send final results to starting process waiting in script_start/0
Parent ! {self(), Result}.
-check_and_install_configfiles(Configs, LogDir, EvHandlers) ->
- case ct_config:check_config_files(Configs) of
- false->
- install([{config,Configs},
- {event_handler,EvHandlers}], LogDir);
- {value, {error, {nofile, File}}} ->
- {error,{cant_read_config_file,File}};
- {value, {error, {wrong_config, Message}}}->
- {error,{wrong_config, Message}};
- {value, {error, {callback, File}}} ->
- {error,{cant_load_callback_module,File}}
+run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
+ case proplists:get_value(refresh_logs, Args) of
+ undefined ->
+ script_start2(StartOpts, Args);
+ Refresh ->
+ LogDir1 = case Refresh of
+ [] -> which_logdir(LogDir);
+ [RefreshDir] -> ?abs(RefreshDir)
+ end,
+ {ok,Cwd} = file:get_cwd(),
+ file:set_cwd(LogDir1),
+ %% give the shell time to print version etc
+ timer:sleep(500),
+ io:nl(),
+ case catch ct_logs:make_all_suites_index(refresh) of
+ {'EXIT',ASReason} ->
+ file:set_cwd(Cwd),
+ {error,{all_suites_index,ASReason}};
+ _ ->
+ case catch ct_logs:make_all_runs_index(refresh) of
+ {'EXIT',ARReason} ->
+ file:set_cwd(Cwd),
+ {error,{all_runs_index,ARReason}};
+ _ ->
+ file:set_cwd(Cwd),
+ io:format("Logs in ~s refreshed!~n~n", [LogDir1]),
+ timer:sleep(500), % time to flush io before quitting
+ ok
+ end
+ end
end.
-script_start2(false, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
- case lists:keysearch(spec, 1, Args) of
- {value,{spec,[]}} ->
+script_start2(StartOpts = #opts{vts = undefined,
+ shell = undefined}, Args) ->
+ TestSpec = proplists:get_value(spec, Args),
+ Opts =
+ case TestSpec of
+ Specs when Specs =/= [], Specs =/= undefined ->
+ %% using testspec as input for test
+ Relaxed = get_start_opt(allow_user_terms, true, false, Args),
+ case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
+ {error,Reason} ->
+ {error,Reason};
+ TS ->
+ SpecStartOpts = get_data_for_node(TS, node()),
+
+ LogDir = choose_val(StartOpts#opts.logdir,
+ SpecStartOpts#opts.logdir),
+
+ Cover = choose_val(StartOpts#opts.cover,
+ SpecStartOpts#opts.cover),
+ MultTT = choose_val(StartOpts#opts.multiply_timetraps,
+ SpecStartOpts#opts.multiply_timetraps),
+ ScaleTT = choose_val(StartOpts#opts.scale_timetraps,
+ SpecStartOpts#opts.scale_timetraps),
+ AllEvHs = merge_vals([StartOpts#opts.event_handlers,
+ SpecStartOpts#opts.event_handlers]),
+ AllInclude = merge_vals([StartOpts#opts.include,
+ SpecStartOpts#opts.include]),
+ application:set_env(common_test, include, AllInclude),
+
+ StartOpts#opts{testspecs = Specs,
+ cover = Cover,
+ logdir = LogDir,
+ config = SpecStartOpts#opts.config,
+ event_handlers = AllEvHs,
+ include = AllInclude,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT}
+ end;
+ _ ->
+ StartOpts
+ end,
+ %% read config/userconfig from start flags
+ InitConfig = ct_config:prepare_config_list(Args),
+ case TestSpec of
+ [] ->
{error,no_testspec_specified};
- {value,{spec,Specs}} ->
- Relaxed = lists:keymember(allow_user_terms, 1, Args),
- %% using testspec as input for test
- case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
- {error,Reason} ->
- {error,Reason};
- TS ->
- {LogDir1,TSCoverFile,ConfigFiles1,EvHandlers1,Include1} =
- get_data_for_node(TS,node()),
- UserInclude =
- case application:get_env(common_test, include) of
- {ok,Include} -> Include++Include1;
- _ -> Include1
- end,
- application:set_env(common_test, include, UserInclude),
- LogDir2 = which_logdir(LogDir,LogDir1),
- CoverOpt = case {Cover,TSCoverFile} of
- {false,undef} -> [];
- {_,undef} -> [Cover];
- {false,_} -> [{cover,TSCoverFile}]
- end,
- case check_and_install_configfiles(
- ConfigFiles++ConfigFiles1, LogDir2,
- EvHandlers++EvHandlers1) of
- ok ->
- {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, CoverOpt, Args, LogDir2);
- Error ->
- Error
- end
+ undefined -> % no testspec used
+ case check_and_install_configfiles(InitConfig,
+ which(logdir,Opts#opts.logdir),
+ Opts#opts.event_handlers) of
+ ok -> % go on read tests from start flags
+ script_start3(Opts#opts{config=InitConfig}, Args);
+ Error ->
+ Error
end;
- false ->
- script_start3(false, ConfigFiles, EvHandlers, Args, LogDir, Cover)
+ _ -> % testspec used
+ %% merge config from start flags with config from testspec
+ AllConfig = merge_vals([InitConfig, Opts#opts.config]),
+ case check_and_install_configfiles(AllConfig,
+ which(logdir,Opts#opts.logdir),
+ Opts#opts.event_handlers) of
+ ok -> % read tests from spec
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ do_run(Run, Skip, Opts#opts{config=AllConfig}, Args);
+ Error ->
+ Error
+ end
end;
-script_start2(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
- script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover).
-script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
- case lists:keysearch(dir, 1, Args) of
- {value,{dir,[]}} ->
- {error,no_dir_specified};
- {value,{dir,Dirs}} ->
- script_start4(VtsOrShell, ConfigFiles, EvHandlers, tests(Dirs),
- Cover, Args, LogDir);
+script_start2(StartOpts, Args) ->
+ script_start3(StartOpts, Args).
+
+check_and_install_configfiles(Configs, LogDir, EvHandlers) ->
+ case ct_config:check_config_files(Configs) of
false ->
- case lists:keysearch(suite, 1, Args) of
- {value,{suite,[]}} ->
+ install([{config,Configs},
+ {event_handler,EvHandlers}], LogDir);
+ {value,{error,{nofile,File}}} ->
+ {error,{cant_read_config_file,File}};
+ {value,{error,{wrong_config,Message}}}->
+ {error,{wrong_config,Message}};
+ {value,{error,{callback,File}}} ->
+ {error,{cant_load_callback_module,File}}
+ end.
+
+script_start3(StartOpts = #opts{cover = Cover}, Args) ->
+ case proplists:get_value(dir, Args) of
+ [] ->
+ {error,no_dir_specified};
+ Dirs when is_list(Dirs) ->
+ script_start4(StartOpts#opts{tests = tests(Dirs)}, Args);
+ undefined ->
+ case proplists:get_value(suite, Args) of
+ [] ->
{error,no_suite_specified};
- {value,{suite,Suites}} ->
- StepOrCover =
- case lists:keysearch(step, 1, Args) of
- {value,Step} -> Step;
- false -> Cover
- end,
- S2M = fun(S) ->
- {filename:dirname(S),
- list_to_atom(
- filename:rootname(filename:basename(S)))}
- end,
- DirMods = lists:map(S2M, Suites),
- {Specified,GroupsAndCases} =
- case {lists:keysearch(group, 1, Args),
- lists:keysearch('case', 1, Args)} of
- {{value,{_,Gs}},{value,{_,Cs}}} -> {true,Gs++Cs};
- {{value,{_,Gs}},_} -> {true,Gs};
- {_,{value,{_,Cs}}} -> {true,Cs};
- _ -> {false,[]}
- end,
- if Specified, length(GroupsAndCases) == 0 ->
- {error,no_case_or_group_specified};
- Specified, length(DirMods) > 1 ->
+ Suites when is_list(Suites) ->
+ StartOpts1 =
+ get_start_opt(step,
+ fun(Step) ->
+ StartOpts#opts{step = Step,
+ cover = undefined}
+ end, StartOpts, Args),
+ DirMods = [suite_to_test(S) || S <- Suites],
+ case groups_and_cases(proplist:get_value(group, Args),
+ proplist:get_value(testcase, Args)) of
+ Error = {error,_} ->
+ Error;
+ [] when DirMods =/= [] ->
+ Ts = tests(DirMods),
+ script_start4(StartOpts1#opts{tests = Ts}, Args);
+ GroupsAndCases when length(DirMods) == 1 ->
+ Ts = tests(DirMods, GroupsAndCases),
+ script_start4(StartOpts1#opts{tests = Ts}, Args);
+ [_,_|_] when length(DirMods) > 1 ->
{error,multiple_suites_and_cases};
- length(GroupsAndCases) > 0, length(DirMods) == 1 ->
- GsAndCs = lists:map(fun(C) -> list_to_atom(C) end,
- GroupsAndCases),
- script_start4(VtsOrShell, ConfigFiles, EvHandlers,
- tests(DirMods, GsAndCs),
- StepOrCover, Args, LogDir);
- not Specified, length(DirMods) > 0 ->
- script_start4(VtsOrShell, ConfigFiles, EvHandlers,
- tests(DirMods),
- StepOrCover, Args, LogDir);
- true ->
- {error,incorrect_suite_and_case_options}
+ _ ->
+ {error,incorrect_suite_option}
end;
- false when VtsOrShell=/=false ->
- script_start4(VtsOrShell, ConfigFiles, EvHandlers,
- [], Cover, Args, LogDir);
- false ->
- script_usage(),
- {error,incorrect_usage}
+ undefined ->
+ if StartOpts#opts.vts ; StartOpts#opts.shell ->
+ script_start4(StartOpts#opts{tests = []}, Args);
+ true ->
+ script_usage(),
+ {error,incorrect_usage}
+ end
end
end.
-script_start4(vts, ConfigFiles, EvHandlers, Tests, false, _Args, LogDir) ->
- vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests);
-script_start4(shell, ConfigFiles, EvHandlers, _Tests, false, Args, LogDir) ->
- Opts = [{config,ConfigFiles},{event_handler,EvHandlers}],
+script_start4(#opts{vts = true, config = Config, event_handler = EvHandlers,
+ tests = Tests, logdir = LogDir}, Args) ->
+ vts:init_data(Config, EvHandlers, ?abs(LogDir), Tests);
+
+script_start4(#opts{shell = true, config = Config, event_handlers = EvHandlers,
+ logdir = LogDir, testspecs = Specs}, Args) ->
+ InstallOpts = [{config,Config},{event_handler,EvHandlers}],
if ConfigFiles == [] ->
ok;
true ->
- io:format("\nInstalling: ~p\n\n", [ConfigFiles])
+ io:format("\nInstalling: ~p\n\n", [Config])
end,
- case install(Opts) of
+ case install(InstallOpts) of
ok ->
ct_util:start(interactive, LogDir),
- log_ts_names(Args),
+ log_ts_names(Specs),
io:nl(),
ok;
Error ->
Error
end;
-script_start4(vts, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) ->
- %% Add support later (maybe).
- script_usage(),
- erlang:halt();
-script_start4(shell, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) ->
- %% Add support later (maybe).
- script_usage();
-script_start4(false, _CfgFs, _EvHs, Tests, Cover={cover,_}, Args, LogDir) ->
- do_run(Tests, [], [Cover], Args, LogDir);
-script_start4(false, _ConfigFiles, _EvHandlers, Tests, false, Args, LogDir) ->
- do_run(Tests, [], [], Args, LogDir);
-script_start4(false, _ConfigFiles, _EvHandlers, Test, Step, Args, LogDir) ->
- do_run(Test, [], [Step], Args, LogDir);
-script_start4(vts, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) ->
- script_usage(),
+
+script_start4(#opts{vts = true, cover = Cover}, _) ->
+ case Cover of
+ undefined ->
+ script_usage();
+ _ ->
+ %% Add support later (maybe).
+ io:format("\nCan't run cover in vts mode.\n\n", [])
+ end,
erlang:halt();
-script_start4(shell, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) ->
- script_usage().
+
+script_start4(#opts{shell = true, cover = Cover}, _) ->
+ case Cover of
+ undefined ->
+ script_usage();
+ _ ->
+ %% Add support later (maybe).
+ io:format("\nCan't run cover in interactive mode.\n\n", [])
+ end;
+
+script_start4(Opts = #opts{tests = Tests}, Args) ->
+ do_run(Tests, [], Opts, Args).
%%%-----------------------------------------------------------------
%%% @spec script_usage() -> ok
@@ -394,8 +412,10 @@ script_usage() ->
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
"\n\t[-dir TestDir1 TestDir2 .. TestDirN] |"
"\n\t[-suite Suite [-case Case]]"
- "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
+ "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
+ "\n\t[-multiply_timetraps]"
+ "\n\t[-scale_timetraps]"
"\n\t[-basic_html]\n\n"),
io:format("Run tests from command line:\n\n"
"\trun_test [-dir TestDir1 TestDir2 .. TestDirN] |"
@@ -409,10 +429,12 @@ script_usage() ->
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
"\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
- "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
+ "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
- "\n\t[-basic_html]"
- "\n\t[-repeat N [-force_stop]] |"
+ "\n\t[-multiply_timetraps]"
+ "\n\t[-scale_timetraps]"
+ "\n\t[-basic_html]"
+ "\n\t[-repeat N [-force_stop]] |"
"\n\t[-duration HHMMSS [-force_stop]] |"
"\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
io:format("Run tests using test specification:\n\n"
@@ -425,10 +447,12 @@ script_usage() ->
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
"\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
- "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
+ "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
- "\n\t[-basic_html]"
- "\n\t[-repeat N [-force_stop]] |"
+ "\n\t[-multiply_timetraps]"
+ "\n\t[-scale_timetraps]"
+ "\n\t[-basic_html]"
+ "\n\t[-repeat N [-force_stop]] |"
"\n\t[-duration HHMMSS [-force_stop]] |"
"\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
io:format("Refresh the HTML index files:\n\n"
@@ -439,7 +463,6 @@ script_usage() ->
"\trun_test -shell"
"\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]\n\n").
-
%%%-----------------------------------------------------------------
%%% @hidden
@@ -462,13 +485,13 @@ install(Opts, LogDir) ->
case whereis(ct_util_server) of
undefined ->
VarFile = variables_file_name(LogDir),
- %% io:format("Varfile=~p~n", [VarFile]),
+ io:format("Varfile=~p~n", [VarFile]),
case file:open(VarFile, [write]) of
{ok,Fd} ->
[io:format(Fd, "~p.\n", [Opt]) || Opt <- Opts],
file:close(Fd),
ok;
- {error,Reason} ->
+ {error,Reason} ->
io:format("CT failed to install configuration data. Please "
"verify that the log directory exists and that "
"write permission is set.\n\n", []),
@@ -487,59 +510,58 @@ variables_file_name(Dir) ->
filename:join(Dir, "variables-"++atom_to_list(node())).
%%%-----------------------------------------------------------------
-%%% @hidden
+%%% @spec run_test(Opts) -> Result
+%%% Opts = [tuple()]
+%%% Result = [TestResult] | {error,Reason}
+%%%
+%%% @doc Start tests from the erlang shell or from an erlang program.
%%% @equiv ct:run_test/1
+%%%-----------------------------------------------------------------
-%% Opts = [OptTuples]
-%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
-%% {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} |
-%% {logdir,LogDir} | {cover,CoverSpecFile} | {step,StepOpts} |
-%% {silent_connections,Conns} | {event_handler,EventHandlers} |
-%% {include,InclDirs} | {auto_compile,Bool} |
-%% {repeat,N} | {duration,DurTime} | {until,StopTime} | {force_stop,Bool} |
-%% {decrypt,KeyOrFile}
-
-run_test(Opt) when is_tuple(Opt) ->
- run_test([Opt]);
-
-run_test(Opts) when is_list(Opts) ->
- case lists:keysearch(refresh_logs, 1, Opts) of
- {value,{_,RefreshDir}} ->
- refresh_logs(?abs(RefreshDir)),
- ok;
- false ->
- Tracing = start_trace(Opts),
+run_test(StartOpt) when is_tuple(StartOpt) ->
+ run_test([StartOpt]);
+
+run_test(StartOpts) when is_list(StartOpts) ->
+ case proplist:get_value(refresh_logs, StartOpts) of
+ undefined ->
+ Tracing = start_trace(StartOpts),
{ok,Cwd} = file:get_cwd(),
- io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
+ io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
Res =
- case ct_repeat:loop_test(func, Opts) of
+ case ct_repeat:loop_test(func, StartOpts) of
false ->
- case catch run_test1(Opts) of
- {'EXIT',Reason} ->
+ case catch run_test1(StartOpts) of
+ {'EXIT',Reason} ->
file:set_cwd(Cwd),
{error,Reason};
- Result ->
+ Result ->
Result
end;
Result ->
Result
end,
stop_trace(Tracing),
- Res
+ Res;
+ RefreshDir ->
+ refresh_logs(?abs(RefreshDir)),
+ ok
end.
-run_test1(Opts) ->
- LogDir =
- case lists:keysearch(logdir, 1, Opts) of
- {value,{_,LD}} when is_list(LD) -> LD;
- false -> "."
- end,
- CfgFiles = ct_config:get_config_file_list(Opts),
+run_test1(StartOpts) ->
+ %% logdir
+ LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end,
+ ".", StartOpts),
+ %% config & userconfig
+ CfgFiles = ct_config:get_config_file_list(StartOpts),
+
+ %% event handlers
EvHandlers =
- case lists:keysearch(event_handler, 1, Opts) of
- {value,{_,H}} when is_atom(H) ->
+ case proplists:get_value(event_handler, StartOpts) of
+ undefined ->
+ [];
+ H when is_atom(H) ->
[{H,[]}];
- {value,{_,H}} ->
+ H ->
Hs =
if is_tuple(H) -> [H];
is_list(H) -> H;
@@ -554,41 +576,34 @@ run_test1(Opts) ->
{EH,Args};
(_) ->
[]
- end, Hs));
- _ ->
- []
- end,
- SilentConns =
- case lists:keysearch(silent_connections, 1, Opts) of
- {value,{_,all}} ->
- [];
- {value,{_,Conns}} ->
- Conns;
- _ ->
- undefined
- end,
- Cover =
- case lists:keysearch(cover, 1, Opts) of
- {value,{_,CoverFile}} ->
- [{cover,?abs(CoverFile)}];
- _ ->
- []
+ end, Hs))
end,
+
+ %% silent connections
+ SilentConns = get_start_opt(silent_connections,
+ fun(all) -> [];
+ (Conns) -> Conns
+ end, StartOpts),
+ %% code coverage
+ Cover = get_start_opt(cover, fun(CoverFile) -> ?abs(CoverFile) end, StartOpts),
+
+ %% timetrap manipulation
+ MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts),
+ ScaleTT = get_start_opt(scale_timetraps, value, false, StartOpts),
+
+ %% auto compile & include files
Include =
- case lists:keysearch(auto_compile, 1, Opts) of
- {value,{auto_compile,ACBool}} ->
- application:set_env(common_test, auto_compile, ACBool),
- [];
- _ ->
+ case proplists:get_value(auto_compile, StartOpts) of
+ undefined ->
application:set_env(common_test, auto_compile, true),
InclDirs =
- case lists:keysearch(include, 1, Opts) of
- {value,{include,Incl}} when is_list(hd(Incl)) ->
- Incl;
- {value,{include,Incl}} when is_list(Incl) ->
- [Incl];
- false ->
- []
+ case proplists:get_value(include, StartOpts) of
+ undefined ->
+ [];
+ Incl when is_list(hd(Incl)) ->
+ Incl;
+ Incl when is_list(Incl) ->
+ [Incl]
end,
case os:getenv("CT_INCLUDE_PATH") of
false ->
@@ -599,10 +614,14 @@ run_test1(Opts) ->
AllInclDirs = InclDirs1++InclDirs,
application:set_env(common_test, include, AllInclDirs),
AllInclDirs
- end
+ end;
+ ACBool ->
+ application:set_env(common_test, auto_compile, ACBool),
+ []
end,
- case lists:keysearch(decrypt, 1, Opts) of
+ %% decrypt config file
+ case lists:keysearch(decrypt, 1, StartOpts) of
{value,{_,Key={key,_}}} ->
application:set_env(common_test, decrypt, Key);
{value,{_,{file,KeyFile}}} ->
@@ -611,39 +630,35 @@ run_test1(Opts) ->
application:unset_env(common_test, decrypt)
end,
- case lists:keysearch(basic_html, 1, Opts) of
- {value,{basic_html,BasicHtmlBool}} ->
- application:set_env(common_test, basic_html, BasicHtmlBool);
- _ ->
- application:set_env(common_test, basic_html, false)
+ %% basic html - used by ct_logs
+ case proplists:get_value(basic_html, StartOpts) of
+ undefined ->
+ application:set_env(common_test, basic_html, false);
+ BasicHtmlBool ->
+ application:set_env(common_test, basic_html, BasicHtmlBool)
end,
- case lists:keysearch(spec, 1, Opts) of
- {value,{_,Specs}} ->
- Relaxed =
- case lists:keysearch(allow_user_terms, 1, Opts) of
- {value,{_,true}} -> true;
- _ -> false
- end,
+ %% stepped execution
+ Step = get_start_opt(step, value, StepOpts),
+
+ Opts = #opts{cover = Cover, step = Step, logdir = LogDir, config = CfgFiles,
+ event_handlers = EvHandlers, include = Include,
+ silent_connections = SilentConns, multiply_timetraps = MultiplyTT,
+ scale_timetraps = ScaleTT},
+
+ %% test specification
+ case proplists:get_value(spec, StartOpts) of
+ undefined ->
+ case proplists:get_value(prepared_tests, StartOpts) of
+ undefined -> % use dir|suite|case
+ run_dir(Opts, StartOpts);
+ {{Run,Skip},Specs} -> % use prepared tests
+ run_prepared(Run, Skip, Opts#opts{testspecs = Specs}, StartOpts)
+ end;
+ Specs ->
+ Relaxed = get_start_opt(allow_user_term, value, false),
%% using testspec(s) as input for test
- run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover,
- replace_opt([{silent_connections,SilentConns}], Opts));
- false ->
- case lists:keysearch(prepared_tests, 1, Opts) of
- {value,{_,{Run,Skip},Specs}} -> % use prepared tests
- run_prepared(LogDir, CfgFiles, EvHandlers,
- Run, Skip, Cover,
- replace_opt([{silent_connections,SilentConns},
- {spec,Specs}],Opts));
- false -> % use dir|suite|case
- StepOrCover =
- case lists:keysearch(step, 1, Opts) of
- {value,Step} -> [Step];
- false -> Cover
- end,
- run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover,
- replace_opt([{silent_connections,SilentConns}], Opts))
- end
+ run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts)
end.
replace_opt([O={Key,_Val}|Os], Opts) ->
@@ -651,83 +666,105 @@ replace_opt([O={Key,_Val}|Os], Opts) ->
replace_opt([], Opts) ->
Opts.
-run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover, Opts) ->
+run_spec_file(Relaxed,
+ Opts = #opts{testspecs = Specs, config = CfgFiles},
+ StartOpts) ->
Specs1 = case Specs of
[X|_] when is_integer(X) -> [Specs];
_ -> Specs
end,
- AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1),
+ AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1),
log_ts_names(AbsSpecs),
case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of
{error,CTReason} ->
exit(CTReason);
TS ->
- {LogDir1,TSCoverFile,CfgFiles1,EvHandlers1,Include1} =
- get_data_for_node(TS, node()),
- application:set_env(common_test, include, Include++Include1),
- LogDir2 = which_logdir(LogDir, LogDir1),
- CoverOpt = case {Cover,TSCoverFile} of
- {[],undef} -> [];
- {_,undef} -> Cover;
- {[],_} -> [{cover,TSCoverFile}]
- end,
- case check_and_install_configfiles(CfgFiles++CfgFiles1, LogDir2,
- EvHandlers++EvHandlers1) of
+ SpecOpts = get_data_for_node(TS, node()),
+ LogDir = choose_val(Opts#opts.logdir,
+ SpecOpts#opts.logdir),
+ AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]),
+ Cover = choose_val(Opts#opts.cover,
+ SpecOpts#opts.cover),
+ MultTT = choose_val(Opts#opts.multiply_timetraps,
+ SpecOpts#opts.multiply_timetraps),
+ ScaleTT = choose_val(Opts#opts.scale_timetraps,
+ SpecOpts#opts.scale_timetraps),
+ AllEvHs = merge_vals([Opts#opts.event_handlers,
+ SpecOpts#opts.event_handlers]),
+ AllInclude = merge_vals([Opts#opts.include,
+ SpecOpts#opts.include]),
+ application:set_env(common_test, include, AllInclude),
+
+ case check_and_install_configfiles(AllConfig,
+ which(logdir,LogDir),
+ AllEvHs) of
ok ->
+ Opts1 = Opts#opts{testspecs = Specs,
+ cover = Cover,
+ logdir = LogDir,
+ config = AllConfig,
+ event_handlers = AllEvHs,
+ include = AllInclude,
+ testspecs = AbsSpecs,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT}
+
{Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, CoverOpt,
- replace_opt([{spec,AbsSpecs}], Opts),
- LogDir2);
+ do_run(Run, Skip, Opts1, StartOpts);
{error,GCFReason} ->
exit(GCFReason)
end
end.
-run_prepared(LogDir, CfgFiles, EvHandlers, Run, Skip, Cover, Opts) ->
+run_prepared(Run, Skip, Opts = #opts{logdir = LogDir,
+ config = CfgFiles,
+ event_handlers = EvHandlers},
+ StartOpts) ->
case check_and_install_configfiles(CfgFiles, LogDir, EvHandlers) of
ok ->
- do_run(Run, Skip, Cover, Opts, LogDir);
+ do_run(Run, Skip, Opts, StartOpts);
{error,Reason} ->
exit(Reason)
- end.
+ end.
check_config_file(Callback, File)->
case Callback:check_parameter(File) of
- {ok, {file, File}}->
+ {ok,{file,File}}->
?abs(File);
- {ok, {config, _}}->
+ {ok,{config,_}}->
File;
- {error, {wrong_config, Message}}->
- exit({wrong_config, {Callback, Message}});
- {error, {nofile, File}}->
- exit({no_such_file, ?abs(File)})
+ {nok,{wrong_config,Message}}->
+ exit({wrong_config,{Callback,Message}});
+ {nok,{nofile,File}}->
+ exit({no_such_file,?abs(File)})
end.
-run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) ->
- AbsCfgFiles =
- lists:map(fun({Callback, FileList})->
- case code:is_loaded(Callback) of
- {file, _Path}->
- ok;
- false->
- case code:load_file(Callback) of
- {module, Callback}->
- ok;
- {error, _}->
- exit({no_such_module, Callback})
- end
- end,
- {Callback,
- lists:map(fun(File)->
- check_config_file(Callback, File)
- end, FileList)}
- end,
- CfgFiles),
+run_dir(Opts = #opts{logdir = LogDir,
+ config = CfgFiles,
+ event_handlers = EvHandlers}, StartOpts) ->
+ AbsCfgFiles =
+ lists:map(fun({Callback,FileList})->
+ case code:is_loaded(Callback) of
+ {file,_Path}->
+ ok;
+ false ->
+ case code:load_file(Callback) of
+ {module,Callback}->
+ ok;
+ {error,_}->
+ exit({no_such_module,Callback})
+ end
+ end,
+ {Callback,
+ lists:map(fun(File)->
+ check_config_file(Callback, File)
+ end, FileList)}
+ end, CfgFiles),
case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir) of
ok -> ok;
{error,IReason} -> exit(IReason)
end,
- case lists:keysearch(dir,1,Opts) of
+ case lists:keysearch(dir, 1, Opts) of
{value,{_,Dirs=[Dir|_]}} when not is_integer(Dir),
length(Dirs)>1 ->
%% multiple dirs (no suite)
@@ -754,12 +791,12 @@ run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) ->
do_run(tests(lists:map(S2M, Suites)), [], StepOrCover, Opts, LogDir);
_ ->
exit(no_tests_specified)
- end;
+ end;
{value,{_,Dir}} ->
case lists:keysearch(suite, 1, Opts) of
{value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) ->
- Mod = if is_atom(Suite) -> Suite;
- true -> list_to_atom(Suite)
+ Mod = if is_atom(Suite) -> Suite;
+ true -> list_to_atom(Suite)
end,
case listify(proplists:get_value(group, Opts, [])) ++
listify(proplists:get_value(testcase, Opts, [])) of
@@ -770,77 +807,90 @@ run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) ->
end;
{value,{_,Suites=[Suite|_]}} when is_list(Suite) ->
Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites),
- do_run(tests(delistify(Dir), Mods), [], StepOrCover, Opts, LogDir);
+ do_run(tests(delistify(Dir), Mods), [], StepOrCover, Opts, LogDir);
{value,{_,Suites}} ->
- do_run(tests(delistify(Dir), Suites), [], StepOrCover, Opts, LogDir);
+ do_run(tests(delistify(Dir), Suites), [], StepOrCover, Opts, LogDir);
false -> % no suite, only dir
do_run(tests(listify(Dir)), [], StepOrCover, Opts, LogDir)
- end
+ end
end.
%%%-----------------------------------------------------------------
-%%% @hidden
+%%% @spec run_testspec(TestSpec) -> Result
+%%% TestSpec = [term()]
%%%
+%%% @doc Run test specified by <code>TestSpec</code>. The terms are
+%%% the same as those used in test specification files.
+%%% @equiv ct:run_testspec/1
+%%%-----------------------------------------------------------------
-%% using testspec(s) as input for test
run_testspec(TestSpec) ->
{ok,Cwd} = file:get_cwd(),
io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
case catch run_testspec1(TestSpec) of
- {'EXIT',Reason} ->
+ {'EXIT',Reason} ->
file:set_cwd(Cwd),
{error,Reason};
- Result ->
+ Result ->
Result
end.
run_testspec1(TestSpec) ->
- case ct_testspec:collect_tests_from_list(TestSpec,false) of
+ case ct_testspec:collect_tests_from_list(TestSpec, false) of
{error,CTReason} ->
exit(CTReason);
TS ->
- {LogDir,TSCoverFile,CfgFiles,EvHandlers,Include} =
- get_data_for_node(TS,node()),
- case os:getenv("CT_INCLUDE_PATH") of
- false ->
- application:set_env(common_test, include, Include);
- CtInclPath ->
- EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]),
- application:set_env(common_test, include, EnvInclude++Include)
- end,
- CoverOpt = if TSCoverFile == undef -> [];
- true -> [{cover,TSCoverFile}]
- end,
- case check_and_install_configfiles(CfgFiles,LogDir,EvHandlers) of
+ Opts = get_data_for_node(TS, node()),
+
+ AllInclude =
+ case os:getenv("CT_INCLUDE_PATH") of
+ false ->
+ Opts#opts.include;
+ CtInclPath ->
+ EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]),
+ EnvInclude++Opts#opts.include,
+ end,
+ application:set_env(common_test, include, AllInclude),
+
+ case check_and_install_configfiles(Opts#opts.config,
+ which(logdir,Opts#opts.logdir),
+ Opts#opts.event_handlers) of
ok ->
- {Run,Skip} = ct_testspec:prepare_tests(TS,node()),
- do_run(Run,Skip,CoverOpt,[],LogDir);
+ Opts1 = Opts#opts{testspecs = [TestSpec],
+ include = AllInclude},
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ do_run(Run, Skip, Opts1, []);
{error,GCFReason} ->
exit(GCFReason)
end
end.
-
get_data_for_node(#testspec{logdir=LogDirs,
cover=CoverFs,
config=Cfgs,
userconfig=UsrCfgs,
event_handler=EvHs,
- include=Incl}, Node) ->
- LogDir = case lists:keysearch(Node,1,LogDirs) of
- {value,{Node,Dir}} -> Dir;
- false -> "."
+ include=Incl,
+ multiply_timetraps=MTs,
+ scale_timetraps=STs}, Node) ->
+ LogDir = case proplists:get_value(Node, LogDirs) of
+ undefined -> ".";
+ Dir -> Dir
end,
- Cover = case lists:keysearch(Node,1,CoverFs) of
- {value,{Node,CovFile}} -> CovFile;
- false -> undef
- end,
- ConfigFiles = [{?ct_config_txt, F} || {N,F} <- Cfgs, N==Node] ++
- [CBF || {N, CBF} <- UsrCfgs, N==Node],
+ Cover = proplists:get_value(Node, CoverFs),
+ MT = proplists:get_value(Node, MTs),
+ ST = proplists:get_value(Node, STs),
+ ConfigFiles = [{?ct_config_txt,F} || {N,F} <- Cfgs, N==Node] ++
+ [CBF || {N,CBF} <- UsrCfgs, N==Node],
EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node],
Include = [I || {N,I} <- Incl, N==Node],
- {LogDir,Cover,ConfigFiles,EvHandlers,Include}.
-
+ #opts{logdir = LogDir,
+ cover = Cover,
+ config = ConfigFiles,
+ event_handlers = EvHandlers,
+ include = Include,
+ multiply_timetraps = MT,
+ scale_timetraps = ST}.
refresh_logs(LogDir) ->
{ok,Cwd} = file:get_cwd(),
@@ -865,11 +915,27 @@ refresh_logs(LogDir) ->
end
end.
-which_logdir(".",Dir) ->
+which(logdir, undefined) ->
+ ".";
+which(logdir, Dir) ->
Dir;
-which_logdir(Dir,_) ->
- Dir.
-
+which(multiply_timetraps, undefined) ->
+ 1;
+which(multiply_timetraps, MT) ->
+ MT;
+which(scale_timetraps, undefined) ->
+ false;
+which(scale_timetraps, ST) ->
+ ST.
+
+choose_val(undefined, V1) ->
+ V1;
+choose_val(V0, _V1) ->
+ V0.
+
+merge_vals(Vs) ->
+ lists:append(Vs).
+
listify([C|_]=Str) when is_integer(C) -> [Str];
listify(L) when is_list(L) -> L;
listify(E) -> [E].
@@ -899,6 +965,22 @@ run(TestDirs) ->
install([]),
do_run(tests(TestDirs), []).
+suite_to_test(Suite) ->
+ {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}.
+
+groups_and_cases(Gs, Cs) when (Gs == undefined ; Gs == []),
+ (Cs == undefined ; Cs == []) ->
+ [];
+groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] ->
+ [list_to_atom(C) || C <- Cs];
+groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] ->
+ [{list_to_atom(G),all} || G <- Gs];
+groups_and_cases([G], Cs) ->
+ [{list_to_atom(G),[list_to_atom(C) || C <- Cs]}];
+groups_and_cases([_,_|_] , Cs) when Cs =/= [] ->
+ {error,multiple_groups_and_cases};
+groups_and_cases(Gs, Cs) ->
+ {error,incorrect_group_or_case_option}.
tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) ->
[{?testdir(TestDir,Suites),ensure_atom(Suites),all}];
@@ -915,30 +997,25 @@ tests(TestDir) when is_list(TestDir), is_integer(hd(TestDir)) ->
tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) ->
[{?testdir(TestDir,all),all,all} || TestDir <- TestDirs].
-do_run(Tests, Opt) ->
- do_run(Tests, [], Opt, [], ".").
-
-do_run(Tests, Opt, LogDir) ->
- do_run(Tests, [], Opt, [], LogDir).
+do_run(Tests, Opt, LogDir) when is_list(Opt) ->
+ do_run(Tests, [], #opts{tests = Tests, logdir = LogDir}, []).
-do_run(Tests, Skip, Opt, Args, LogDir) ->
+do_run(Tests, Skip, Opts, Args) ->
+ #opts{cover = Cover} = Opts
case code:which(test_server) of
non_existing ->
exit({error,no_path_to_test_server});
_ ->
- Opt1 =
- case lists:keysearch(cover, 1, Opt) of
- {value,{_,CoverFile}} ->
- case ct_cover:get_spec(CoverFile) of
- {error,Reason} ->
- exit({error,Reason});
- Spec ->
- [{cover_spec,Spec} |
- lists:keydelete(cover, 1, Opt)]
- end;
- _ ->
- Opt
- end,
+ Opts1 = if Cover == undefined ->
+ Opts;
+ true ->
+ case ct_cover:get_spec(Cover) of
+ {error,Reason} ->
+ exit({error,Reason});
+ CoverSpec ->
+ Opts#opts{coverspec = CoverSpec}
+ end
+ end,
%% This env variable is used by test_server to determine
%% which framework it runs under.
case os:getenv("TEST_SERVER_FRAMEWORK") of
@@ -956,7 +1033,6 @@ do_run(Tests, Skip, Opt, Args, LogDir) ->
"To enter the interactive mode again, "
"run ct:start_interactive()\n\n",[]),
{error,interactive_mode};
-
_Pid ->
%% save style sheet info
case lists:keysearch(stylesheet, 1, Args) of
@@ -982,7 +1058,7 @@ do_run(Tests, Skip, Opt, Args, LogDir) ->
_ ->
ok
end,
- log_ts_names(Args),
+ log_ts_names(Opts1#opts.testspecs),
TestSuites = suite_tuples(Tests),
{SuiteMakeErrors,AllMakeErrors} =
@@ -1000,7 +1076,7 @@ do_run(Tests, Skip, Opt, Args, LogDir) ->
SavedErrors = save_make_errors(SuiteMakeErrors),
ct_repeat:log_loop_info(Args),
{Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors),
- R = do_run_test(Tests1, Skip1, Opt1),
+ R = do_run_test(Tests1, Skip1, Opts1),
ct_util:stop(normal),
R;
false ->
@@ -1026,7 +1102,7 @@ auto_compile(TestSuites) ->
case application:get_env(common_test, include) of
{ok,UserInclDirs} when length(UserInclDirs) > 0 ->
io:format("Including the following directories:~n"),
- [begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end ||
+ [begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end ||
UserInclDir <- UserInclDirs];
_ ->
[]
@@ -1034,11 +1110,11 @@ auto_compile(TestSuites) ->
SuiteMakeErrors =
lists:flatmap(fun({TestDir,Suite} = TS) ->
case run_make(suites, TestDir, Suite, UserInclude) of
- {error,{make_failed,Bad}} ->
+ {error,{make_failed,Bad}} ->
[{TS,Bad}];
- {error,_} ->
+ {error,_} ->
[{TS,[filename:join(TestDir,"*_SUITE")]}];
- _ ->
+ _ ->
[]
end
end, TestSuites),
@@ -1062,13 +1138,13 @@ auto_compile(TestSuites) ->
true -> % already visited
{Done,Failed}
end
- end, {[],[]}, TestSuites),
+ end, {[],[]}, TestSuites),
{SuiteMakeErrors,lists:reverse(HelpMakeErrors)}.
%% verify that specified test suites exist (if auto compile is disabled)
verify_suites(TestSuites) ->
io:nl(),
- Verify =
+ Verify =
fun({Dir,Suite},NotFound) ->
case locate_test_dir(Dir, Suite) of
{ok,TestDir} ->
@@ -1077,11 +1153,11 @@ verify_suites(TestSuites) ->
true ->
Beam = filename:join(TestDir, atom_to_list(Suite)++".beam"),
case filelib:is_regular(Beam) of
- true ->
+ true ->
NotFound;
- false ->
+ false ->
Name = filename:join(TestDir, atom_to_list(Suite)),
- io:format("Suite ~w not found in directory ~s~n",
+ io:format("Suite ~w not found in directory ~s~n",
[Suite,TestDir]),
[{{Dir,Suite},[Name]} | NotFound]
end
@@ -1093,8 +1169,7 @@ verify_suites(TestSuites) ->
end
end,
lists:reverse(lists:foldl(Verify, [], TestSuites)).
-
-
+
save_make_errors([]) ->
[];
save_make_errors(Errors) ->
@@ -1110,7 +1185,7 @@ get_bad_suites([{{_TestDir,_Suite},Failed}|Errors], BadSuites) ->
get_bad_suites([], BadSuites) ->
BadSuites.
-
+
%%%-----------------------------------------------------------------
%%% @hidden
@@ -1121,7 +1196,7 @@ step(TestDir, Suite, Case) ->
%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:step/4
-step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case),
+step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case),
Suite =/= all, Case =/= all ->
do_run([{TestDir,Suite,Case}], [{step,Opts}]).
@@ -1154,7 +1229,7 @@ final_tests([{TestDir,Suites,_}|Tests],
Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites,
S == S1, TD == TestDir],
- Final1 = [{TestDir,S,all} || S <- Suites],
+ Final1 = [{TestDir,S,all} || S <- Suites],
final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);
final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
@@ -1187,7 +1262,7 @@ final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) ->
final_tests([], Final, Skip, _Bad) ->
{lists:reverse(Final),Skip}.
-continue([]) ->
+continue([]) ->
true;
continue(_MakeErrors) ->
io:nl(),
@@ -1228,7 +1303,7 @@ set_group_leader_same_as_shell() ->
false
end
end,
- case [P || P <- processes(), GS2or3(P),
+ case [P || P <- processes(), GS2or3(P),
true == lists:keymember(shell,1,element(2,process_info(P,dictionary)))] of
[GL|_] ->
group_leader(GL, self());
@@ -1252,29 +1327,29 @@ check_and_add([{TestDir0,M,_} | Tests], Added) ->
check_and_add([], _) ->
ok.
-do_run_test(Tests, Skip, Opt) ->
+do_run_test(Tests, Skip, Opts) ->
case check_and_add(Tests, []) of
ok ->
ct_util:set_testdata({stats,{0,0,{0,0}}}),
ct_util:set_testdata({cover,undefined}),
test_server_ctrl:start_link(local),
- case lists:keysearch(cover_spec, 1, Opt) of
- {value,{_,CovData={CovFile,
- CovNodes,
- _CovImport,
- CovExport,
- #cover{app = CovApp,
- level = CovLevel,
- excl_mods = CovExcl,
- incl_mods = CovIncl,
- cross = CovCross,
- src = _CovSrc}}}} ->
+ case Opts#opts.coverspec of
+ CovData={CovFile,
+ CovNodes,
+ _CovImport,
+ CovExport,
+ #cover{app = CovApp,
+ level = CovLevel,
+ excl_mods = CovExcl,
+ incl_mods = CovIncl,
+ cross = CovCross,
+ src = _CovSrc}} ->
ct_logs:log("COVER INFO","Using cover specification file: ~s~n"
"App: ~w~n"
"Cross cover: ~w~n"
"Including ~w modules~n"
"Excluding ~w modules",
- [CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]),
+ [CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]),
%% cover export file will be used for export and import
%% between tests so make sure it doesn't exist initially
@@ -1307,33 +1382,37 @@ do_run_test(Tests, Skip, Opt) ->
true;
_ ->
false
- end,
+ end,
%% let test_server expand the test tuples and count no of cases
{Suites,NoOfCases} = count_test_cases(Tests, Skip),
Suites1 = delete_dups(Suites),
NoOfTests = length(Tests),
NoOfSuites = length(Suites1),
- ct_util:warn_duplicates(Suites1),
+ ct_util:warn_duplicates(Suites1),
{ok,Cwd} = file:get_cwd(),
io:format("~nCWD set to: ~p~n", [Cwd]),
if NoOfCases == unknown ->
- io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n",
+ io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n",
[NoOfTests,NoOfSuites]),
- ct_logs:log("TEST INFO","~w test(s), ~w suite(s)",
+ ct_logs:log("TEST INFO","~w test(s), ~w suite(s)",
[NoOfTests,NoOfSuites]);
true ->
- io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n",
+ io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n",
[NoOfTests,NoOfCases,NoOfSuites]),
- ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)",
+ ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)",
[NoOfTests,NoOfCases,NoOfSuites])
end,
+
+ test_server_ctrl:multiply_timetraps(Opts#opts.multiply_timetraps),
+ test_server_ctrl:scale_timetraps(Opts#opts.scale_timetraps),
+
ct_event:notify(#event{name=start_info,
node=node(),
data={NoOfTests,NoOfSuites,NoOfCases}}),
- CleanUp = add_jobs(Tests, Skip, Opt, []),
+ CleanUp = add_jobs(Tests, Skip, Opts, []),
unlink(whereis(test_server_ctrl)),
- catch test_server_ctrl:wait_finish(),
- %% check if last testcase has left a "dead" trace window
+ catch test_server_ctrl:wait_finish(),
+ %% check if last testcase has left a "dead" trace window
%% behind, and if so, kill it
case ct_util:get_testdata(interpret) of
{_What,kill,{TCPid,AttPid}} ->
@@ -1341,8 +1420,8 @@ do_run_test(Tests, Skip, Opt) ->
_ ->
ok
end,
- lists:foreach(fun(Suite) ->
- maybe_cleanup_interpret(Suite, Opt)
+ lists:foreach(fun(Suite) ->
+ maybe_cleanup_interpret(Suite, Opt)
end, CleanUp);
Error ->
Error
@@ -1358,7 +1437,7 @@ count_test_cases(Tests, Skip) ->
SendResult = fun(Me, Result) -> Me ! {no_of_cases,Result} end,
TSPid = test_server_ctrl:start_get_totals(SendResult),
Ref = erlang:monitor(process, TSPid),
- add_jobs(Tests, Skip, [], []),
+ add_jobs(Tests, Skip, #opts{}, []),
{Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref),
erlang:demonitor(Ref),
test_server_ctrl:stop_get_totals(),
@@ -1368,11 +1447,11 @@ count_test_cases1(0, N, Suites, _) ->
{lists:flatten(Suites), N};
count_test_cases1(Jobs, N, Suites, Ref) ->
receive
- {no_of_cases,{Ss,N1}} ->
+ {no_of_cases,{Ss,N1}} ->
count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref);
- {'DOWN', Ref, _, _, _} ->
+ {'DOWN', Ref, _, _, _} ->
{[],0}
- end.
+ end.
add_known(unknown, _) ->
unknown;
@@ -1381,72 +1460,72 @@ add_known(_, unknown) ->
add_known(N, N1) ->
N+N1.
-add_jobs([{TestDir,all,_}|Tests], Skip, Opt, CleanUp) ->
+add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) ->
Name = get_name(TestDir),
case catch test_server_ctrl:add_dir_with_skip(Name, TestDir,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, CleanUp)
+ add_jobs(Tests, Skip, Opts, CleanUp)
end;
-add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opt, CleanUp) when is_atom(Suite) ->
- add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp);
-add_jobs([{TestDir,Suites,all}|Tests], Skip, Opt, CleanUp) when is_list(Suites) ->
+add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) ->
+ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp);
+add_jobs([{TestDir,Suites,all}|Tests], Skip, Opts, CleanUp) when is_list(Suites) ->
Name = get_name(TestDir) ++ ".suites",
case catch test_server_ctrl:add_module_with_skip(Name, Suites,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, CleanUp)
+ add_jobs(Tests, Skip, Opts, CleanUp)
end;
-add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp) ->
- case maybe_interpret(Suite, all, Opt) of
+add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) ->
+ case maybe_interpret(Suite, all, Opts) of
ok ->
Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite),
case catch test_server_ctrl:add_module_with_skip(Name, [Suite],
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
+ add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
end;
Error ->
Error
end;
-add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opt, CleanUp) when is_atom(Case) ->
- add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp);
-add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opt, CleanUp) when is_list(Cases) ->
- case maybe_interpret(Suite, Cases, Opt) of
+add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) ->
+ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp);
+add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opts, CleanUp) when is_list(Cases) ->
+ case maybe_interpret(Suite, Cases, Opts) of
ok ->
Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases",
case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
+ add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
end;
Error ->
Error
end;
-add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp) when is_atom(Case) ->
- case maybe_interpret(Suite, Case, Opt) of
+add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) ->
+ case maybe_interpret(Suite, Case, Opts) of
ok ->
- Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++
+ Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++
atom_to_list(Case),
case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
+ add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
end;
Error ->
Error
@@ -1496,7 +1575,7 @@ get_name(Dir) ->
end,
Base = filename:basename(TestDir),
case filename:basename(filename:dirname(TestDir)) of
- "" ->
+ "" ->
Base;
TopDir ->
TopDir ++ "." ++ Base
@@ -1527,10 +1606,10 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
{i,CtInclude},
{i,XmerlInclude},
debug_info],
- Result =
+ Result =
if Mod == all ; Targets == helpmods ->
case (catch ct_make:all([noexec|ErlFlags])) of
- {'EXIT',_} = Failure ->
+ {'EXIT',_} = Failure ->
Failure;
MakeInfo ->
FileTest = fun(F, suites) -> is_suite(F);
@@ -1549,7 +1628,7 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
true ->
(catch ct_make:files([Mod], [load|ErlFlags]))
end,
-
+
ok = file:set_cwd(Cwd),
%% send finished_make notification
ct_event:notify(#event{name=finished_make,
@@ -1563,7 +1642,7 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
{error,{make_crashed,TestDir,Reason}};
{error,ModInfo} ->
io:format("{error,make_failed}\n", []),
- Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo,
+ Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo,
R == error],
{error,{make_failed,Bad}}
end;
@@ -1575,8 +1654,8 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
get_dir(App, Dir) ->
filename:join(code:lib_dir(App), Dir).
-maybe_interpret(Suite, Cases, [{step,StepOpts}]) ->
- %% if other suite has run before this one, check if last testcase
+maybe_interpret(Suite, Cases, #opts{step = StepOpts}) when StepOpts =/= undefined ->
+ %% if other suite has run before this one, check if last testcase
%% has left a "dead" trace window behind, and if so, kill it
case ct_util:get_testdata(interpret) of
{_What,kill,{TCPid,AttPid}} ->
@@ -1619,7 +1698,7 @@ maybe_interpret2(Suite, Cases, StepOpts) ->
WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of
true -> no_kill;
false -> kill
- end,
+ end,
ct_util:set_testdata({interpret,{{Suite,Cases},WinOp,
{undefined,undefined}}}),
ok.
@@ -1640,18 +1719,15 @@ maybe_cleanup_interpret(Suite, [{step,_}]) ->
maybe_cleanup_interpret(_, _) ->
ok.
-log_ts_names(Args) ->
- case lists:keysearch(spec, 1, Args) of
- {value,{_,Specs}} ->
- List = lists:map(fun(Name) ->
- Name ++ " "
- end, Specs),
- ct_logs:log("Test Specification file(s)", "~s",
- [lists:flatten(List)]);
- _ ->
- ok
- end.
-
+log_ts_names([]) ->
+ ok;
+log_ts_names(Specs) ->
+ List = lists:map(fun(Name) ->
+ Name ++ " "
+ end, Specs),
+ ct_logs:log("Test Specification file(s)", "~s",
+ [lists:flatten(List)]).
+
merge_arguments(Args) ->
merge_arguments(Args, []).
@@ -1659,6 +1735,8 @@ merge_arguments([LogDir={logdir,_}|Args], Merged) ->
merge_arguments(Args, handle_arg(replace, LogDir, Merged));
merge_arguments([CoverFile={cover,_}|Args], Merged) ->
merge_arguments(Args, handle_arg(replace, CoverFile, Merged));
+merge_arguments([{'case',TC}|Args], Merged) ->
+ merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged));
merge_arguments([Arg={_,_}|Args], Merged) ->
merge_arguments(Args, handle_arg(merge, Arg, Merged));
merge_arguments([], Merged) ->
@@ -1673,6 +1751,23 @@ handle_arg(Op, Arg, [Other|Merged]) ->
handle_arg(_,Arg,[]) ->
[Arg].
+get_start_opt(Key, IfExists, Args) ->
+ get_start_opt(Key, IfExists, undefined, Args);
+
+get_start_opt(Key, IfExists, IfNotExists, Args) ->
+ case lists:keysearch(Key, 1, Args) of
+ {value,{Key,Val}} when is_function(IfExists) ->
+ IfExists(Val);
+ {value,{Key,Val}} when IfExists == value ->
+ Val;
+ {value,{Key,_Val}} ->
+ IfExists;
+ _ when is_function(IfNotExists) ->
+ IfNotExists();
+ _ ->
+ IfNotExists
+ end.
+
locate_test_dir(Dir, Suite) ->
TestDir = case ct_util:is_test_dir(Dir) of
true -> Dir;
@@ -1737,18 +1832,18 @@ start_trace(Args) ->
case file:consult(TraceSpec) of
{ok,Terms} ->
case catch do_trace(Terms) of
- ok ->
+ ok ->
true;
{_,Error} ->
io:format("Warning! Tracing not started. Reason: ~p~n~n",
[Error]),
false
- end;
+ end;
{_,Error} ->
io:format("Warning! Tracing not started. Reason: ~p~n~n",
[Error]),
false
- end;
+ end;
false ->
false
end.
@@ -1760,22 +1855,22 @@ do_trace(Terms) ->
case dbg:tpl(M,[{'_',[],[{return_trace}]}]) of
{error,What} -> exit({error,{tracing_failed,What}});
_ -> ok
- end;
+ end;
({f,M,F}) ->
case dbg:tpl(M,F,[{'_',[],[{return_trace}]}]) of
{error,What} -> exit({error,{tracing_failed,What}});
_ -> ok
- end;
+ end;
(Huh) ->
exit({error,{unrecognized_trace_term,Huh}})
end, Terms),
ok.
-
+
stop_trace(true) ->
dbg:stop_clear();
stop_trace(false) ->
ok.
-
+
ensure_atom(Atom) when is_atom(Atom) ->
Atom;
ensure_atom(String) when is_list(String), is_integer(hd(String)) ->
@@ -1784,4 +1879,3 @@ ensure_atom(List) when is_list(List) ->
[ensure_atom(Item) || Item <- List];
ensure_atom(Other) ->
Other.
-