aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/ct_run.erl
diff options
context:
space:
mode:
authorErlang/OTP <[email protected]>2010-06-11 09:08:08 +0000
committerErlang/OTP <[email protected]>2010-06-11 09:08:08 +0000
commitc0f23851df50d6215981805102754b590c05acec (patch)
tree5a202fe2abc7cd77e9808930758b81d05957fab6 /lib/common_test/src/ct_run.erl
parenta28aa321007f62ae4dce415cd190778c3ac184e7 (diff)
parent2d7ba88ebbb59a473dbcefd0f9dee1f1b816935e (diff)
downloadotp-c0f23851df50d6215981805102754b590c05acec.tar.gz
otp-c0f23851df50d6215981805102754b590c05acec.tar.bz2
otp-c0f23851df50d6215981805102754b590c05acec.zip
Merge branch 'peppe/common_test_r14_dev_2' into dev
* peppe/common_test_r14_dev_2: Misc documentation updates Add documentation for run_test program Step vsn for test_server to 3.4 Make {repeat*,N} property in group execute the group N times exactly Fix so that ct_run converts relative diretories in the code path to absolute Fix bug in handling framework:end_tc timeouts Fix bug that prevents the interactive shell mode to start properly Fix failing multiply timetrap test case Minor fixes in code and test suites Add support for executing pre-loaded suites (e.g. modules loaded as binaries) Add test suite for remote loading of binary suites Fix error with {repeat,0} property in groups causing double iterations Add support for config info functions (e.g. init_per_suite/0) Add support for dynamic timetrap handling Have end_per_testcase run even after timetrap_timeout and abort_testcase Flush old DOWN messages in demonitor Add groups in test specifications Add new tests for test case groups and test specifications Improve and fix various test suites Add event_handler_init start flag that can pass init arguments to event handlers ... OTP-8703 peppe/common_test_r14_dev_2
Diffstat (limited to 'lib/common_test/src/ct_run.erl')
-rw-r--r--lib/common_test/src/ct_run.erl1682
1 files changed, 1010 insertions, 672 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 6b1063f74c..6a9c42d1b9 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -24,7 +24,6 @@
-module(ct_run).
-
%% Script interface
-export([script_start/0,script_usage/0]).
@@ -46,11 +45,27 @@
-define(abs(Name), filename:absname(Name)).
-define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)).
+-record(opts, {vts,
+ shell,
+ cover,
+ coverspec,
+ step,
+ logdir,
+ config = [],
+ event_handlers = [],
+ include = [],
+ silent_connections,
+ stylesheet,
+ multiply_timetraps = 1,
+ scale_timetraps = false,
+ testspecs = [],
+ tests}).
+
%%%-----------------------------------------------------------------
%%% @spec script_start() -> void()
%%%
-%%% @doc Start tests via the run_test script.
-%%%
+%%% @doc Start tests via the run_test program or script.
+%%%
%%% <p>Example:<br/><code>./run_test -config config.ctc -dir
%%% $TEST_DIR</code></p>
%%%
@@ -59,13 +74,53 @@
%%%
script_start() ->
process_flag(trap_exit, true),
- Args = merge_arguments(init:get_arguments()),
+ Init = init:get_arguments(),
+ CtArgs = lists:takewhile(fun({ct_erl_args,_}) -> false;
+ (_) -> true end, Init),
+
+ %% convert relative dirs added with pa or pz (pre erl_args on
+ %% the run_test command line) to absolute so that app modules
+ %% can be found even after CT changes CWD to logdir
+ rel_to_abs(CtArgs),
+
+ Args =
+ case application:get_env(common_test, run_test_start_opts) of
+ {ok,EnvStartOpts} ->
+ FlagFilter = fun(Flags) ->
+ lists:filter(fun({root,_}) -> false;
+ ({progname,_}) -> false;
+ ({home,_}) -> false;
+ ({noshell,_}) -> false;
+ ({noinput,_}) -> false;
+ (_) -> true
+ end, Flags)
+ end,
+ %% used for purpose of testing the run_test interface
+ io:format(user, "~n-------------------- START ARGS --------------------~n", []),
+ io:format(user, "--- Init args:~n~p~n", [FlagFilter(Init)]),
+ io:format(user, "--- CT args:~n~p~n", [FlagFilter(CtArgs)]),
+ EnvArgs = opts2args(EnvStartOpts),
+ io:format(user, "--- Env opts -> args:~n~p~n =>~n~p~n",
+ [EnvStartOpts,EnvArgs]),
+ Merged = merge_arguments(CtArgs ++ EnvArgs),
+ io:format(user, "--- Merged args:~n~p~n", [FlagFilter(Merged)]),
+ io:format(user, "----------------------------------------------------~n~n", []),
+ Merged;
+ _ ->
+ merge_arguments(CtArgs)
+ end,
+ case proplists:get_value(help, Args) of
+ undefined -> script_start(Args);
+ _ -> script_usage()
+ end.
+
+script_start(Args) ->
Tracing = start_trace(Args),
- Res =
+ Res =
case ct_repeat:loop_test(script, Args) of
- false ->
+ false ->
{ok,Cwd} = file:get_cwd(),
- CTVsn =
+ CTVsn =
case filename:basename(code:lib_dir(common_test)) of
CTBase when is_list(CTBase) ->
case string:tokens(CTBase, "-") of
@@ -76,7 +131,7 @@ script_start() ->
io:format("~nCommon Test~s starting (cwd is ~s)~n~n", [CTVsn,Cwd]),
Self = self(),
Pid = spawn_link(fun() -> script_start1(Self, Args) end),
- receive
+ receive
{'EXIT',Pid,Reason} ->
case Reason of
{user_error,What} ->
@@ -101,293 +156,320 @@ script_start() ->
Res.
script_start1(Parent, Args) ->
- case lists:keymember(preload, 1, Args) of
- true -> preload();
- false -> ok
- end,
-
- VtsOrShell =
- case lists:keymember(vts, 1, Args) of
- true ->
- vts;
- false ->
- case lists:keymember(shell, 1, Args) of
- true -> shell;
- false -> false
- end
- end,
- LogDir =
- case lists:keysearch(logdir, 1, Args) of
- {value,{logdir,[LogD]}} -> LogD;
- false -> "."
- end,
- EvHandlers =
- case lists:keysearch(event_handler, 1, Args) of
- {value,{event_handler,Handlers}} ->
- lists:map(fun(H) -> {list_to_atom(H),[]} end, Handlers);
- false ->
- []
- end,
- Cover =
- case lists:keysearch(cover, 1, Args) of
- {value,{cover,CoverFile}} ->
- {cover,?abs(CoverFile)};
- false ->
- false
- end,
-
- case lists:keysearch(ct_decrypt_key, 1, Args) of
- {value,{_,[DecryptKey]}} ->
+ %% read general start flags
+ Vts = get_start_opt(vts, true, Args),
+ Shell = get_start_opt(shell, true, Args),
+ Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args),
+ LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, ".", Args),
+ MultTT = get_start_opt(multiply_timetraps,
+ fun([MT]) -> list_to_integer(MT) end, 1, Args),
+ ScaleTT = get_start_opt(scale_timetraps,
+ fun([CT]) -> list_to_atom(CT);
+ ([]) -> true
+ end, false, Args),
+ EvHandlers = event_handler_args2opts(Args),
+
+ %% check flags and set corresponding application env variables
+
+ %% ct_decrypt_key | ct_decrypt_file
+ case proplists:get_value(ct_decrypt_key, Args) of
+ [DecryptKey] ->
application:set_env(common_test, decrypt, {key,DecryptKey});
- false ->
- case lists:keysearch(ct_decrypt_file, 1, Args) of
- {value,{_,[DecryptFile]}} ->
- application:set_env(common_test, decrypt,
- {file,filename:absname(DecryptFile)});
- false ->
+ undefined ->
+ case proplists:get_value(ct_decrypt_file, Args) of
+ [DecryptFile] ->
+ application:set_env(common_test, decrypt,
+ {file,?abs(DecryptFile)});
+ undefined ->
application:unset_env(common_test, decrypt)
end
end,
-
- case lists:keysearch(no_auto_compile, 1, Args) of
- {value,_} ->
- application:set_env(common_test, auto_compile, false);
- false ->
- application:set_env(common_test, auto_compile, true),
-
- InclDirs =
- case lists:keysearch(include,1,Args) of
- {value,{include,Incl}} when is_list(hd(Incl)) ->
- Incl;
- {value,{include,Incl}} when is_list(Incl) ->
- [Incl];
+ %% no_auto_compile + include
+ IncludeDirs =
+ case proplists:get_value(no_auto_compile, Args) of
+ undefined ->
+ application:set_env(common_test, auto_compile, true),
+ InclDirs =
+ case proplists:get_value(include, Args) of
+ Incl when is_list(hd(Incl)) ->
+ Incl;
+ Incl when is_list(Incl) ->
+ [Incl];
+ undefined ->
+ []
+ end,
+ case os:getenv("CT_INCLUDE_PATH") of
false ->
- []
- end,
- case os:getenv("CT_INCLUDE_PATH") of
- false ->
- application:set_env(common_test, include, InclDirs);
- CtInclPath ->
- InclDirs1 = string:tokens(CtInclPath,[$:,$ ,$,]),
- application:set_env(common_test, include, InclDirs1++InclDirs)
- end
- end,
-
- case lists:keysearch(basic_html, 1, Args) of
- {value,_} ->
- application:set_env(common_test, basic_html, true);
- false ->
- application:set_env(common_test, basic_html, false)
- end,
-
- Result =
- case lists:keysearch(refresh_logs, 1, Args) of
- {value,{refresh_logs,Refresh}} ->
- LogDir1 = case Refresh of
- [] -> LogDir;
- [RefreshDir] -> ?abs(RefreshDir)
- end,
- {ok,Cwd} = file:get_cwd(),
- file:set_cwd(LogDir1),
- timer:sleep(500), % give the shell time to print version etc
- io:nl(),
- case catch ct_logs:make_all_suites_index(refresh) of
- {'EXIT',ASReason} ->
- file:set_cwd(Cwd),
- {error,{all_suites_index,ASReason}};
- _ ->
- case catch ct_logs:make_all_runs_index(refresh) of
- {'EXIT',ARReason} ->
- file:set_cwd(Cwd),
- {error,{all_runs_index,ARReason}};
- _ ->
- file:set_cwd(Cwd),
- io:format("Logs in ~s refreshed!~n~n", [LogDir1]),
- timer:sleep(500), % time to flush io before quitting
- ok
- end
+ application:set_env(common_test, include, InclDirs),
+ InclDirs;
+ CtInclPath ->
+ AllInclDirs =
+ string:tokens(CtInclPath,[$:,$ ,$,]) ++ InclDirs,
+ application:set_env(common_test, include, AllInclDirs),
+ AllInclDirs
end;
- false ->
- case lists:keysearch(ct_config, 1, Args) of
- {value,{ct_config,ConfigFiles}} ->
- case lists:keysearch(spec, 1, Args) of
- false ->
- case get_configfiles(ConfigFiles, [], LogDir,
- EvHandlers) of
- ok ->
- script_start2(VtsOrShell, ConfigFiles,
- EvHandlers, Args, LogDir,
- Cover);
- Error ->
- Error
- end;
- _ ->
- script_start2(VtsOrShell, ConfigFiles,
- EvHandlers, Args, LogDir, Cover)
- end;
- false ->
- case install([{config,[]},
- {event_handler,EvHandlers}],
- LogDir) of
- ok ->
- script_start2(VtsOrShell, [], EvHandlers,
- Args, LogDir, Cover);
- Error ->
- Error
- end
- end
+ _ ->
+ application:set_env(common_test, auto_compile, false),
+ []
end,
+ %% silent connections
+ SilentConns =
+ get_start_opt(silent_connections,
+ fun(["all"]) -> [];
+ (Conns) -> [list_to_atom(Conn) || Conn <- Conns]
+ end, Args),
+ %% stylesheet
+ Stylesheet = get_start_opt(stylesheet,
+ fun([SS]) -> ?abs(SS) end, Args),
+ %% basic_html - used by ct_logs
+ case proplists:get_value(basic_html, Args) of
+ undefined ->
+ application:set_env(common_test, basic_html, false);
+ _ ->
+ application:set_env(common_test, basic_html, true)
+ end,
+
+ StartOpts = #opts{vts = Vts, shell = Shell, cover = Cover,
+ logdir = LogDir, event_handlers = EvHandlers,
+ include = IncludeDirs,
+ silent_connections = SilentConns,
+ stylesheet = Stylesheet,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT},
+
+ %% check if log files should be refreshed or go on to run tests...
+ Result = run_or_refresh(StartOpts, Args),
+ %% send final results to starting process waiting in script_start/0
Parent ! {self(), Result}.
-get_configfiles([File|Files], Acc, LogDir, EvHandlers) ->
- case filelib:is_file(File) of
- true ->
- get_configfiles(Files, [?abs(File)|Acc],
- LogDir, EvHandlers);
- false ->
- {error,{cant_read_config_file,File}}
- end;
-get_configfiles([], Acc, LogDir, EvHandlers) ->
- install([{config,lists:reverse(Acc)}, {event_handler,EvHandlers}], LogDir).
+run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
+ case proplists:get_value(refresh_logs, Args) of
+ undefined ->
+ script_start2(StartOpts, Args);
+ Refresh ->
+ LogDir1 = case Refresh of
+ [] -> which(logdir,LogDir);
+ [RefreshDir] -> ?abs(RefreshDir)
+ end,
+ {ok,Cwd} = file:get_cwd(),
+ file:set_cwd(LogDir1),
+ %% give the shell time to print version etc
+ timer:sleep(500),
+ io:nl(),
+ case catch ct_logs:make_all_suites_index(refresh) of
+ {'EXIT',ASReason} ->
+ file:set_cwd(Cwd),
+ {error,{all_suites_index,ASReason}};
+ _ ->
+ case catch ct_logs:make_all_runs_index(refresh) of
+ {'EXIT',ARReason} ->
+ file:set_cwd(Cwd),
+ {error,{all_runs_index,ARReason}};
+ _ ->
+ file:set_cwd(Cwd),
+ io:format("Logs in ~s refreshed!~n~n", [LogDir1]),
+ timer:sleep(500), % time to flush io before quitting
+ ok
+ end
+ end
+ end.
-script_start2(false, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
- case lists:keysearch(spec, 1, Args) of
- {value,{spec,[]}} ->
+script_start2(StartOpts = #opts{vts = undefined,
+ shell = undefined}, Args) ->
+ TestSpec = proplists:get_value(spec, Args),
+ {Terms,Opts} =
+ case TestSpec of
+ Specs when Specs =/= [], Specs =/= undefined ->
+ %% using testspec as input for test
+ Relaxed = get_start_opt(allow_user_terms, true, false, Args),
+ case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
+ {E,Reason} when E == error ; E == 'EXIT' ->
+ {{error,Reason},StartOpts};
+ TS ->
+ SpecStartOpts = get_data_for_node(TS, node()),
+
+ LogDir = choose_val(StartOpts#opts.logdir,
+ SpecStartOpts#opts.logdir),
+
+ Cover = choose_val(StartOpts#opts.cover,
+ SpecStartOpts#opts.cover),
+ MultTT = choose_val(StartOpts#opts.multiply_timetraps,
+ SpecStartOpts#opts.multiply_timetraps),
+ ScaleTT = choose_val(StartOpts#opts.scale_timetraps,
+ SpecStartOpts#opts.scale_timetraps),
+ AllEvHs = merge_vals([StartOpts#opts.event_handlers,
+ SpecStartOpts#opts.event_handlers]),
+ AllInclude = merge_vals([StartOpts#opts.include,
+ SpecStartOpts#opts.include]),
+ application:set_env(common_test, include, AllInclude),
+
+ {TS,StartOpts#opts{testspecs = Specs,
+ cover = Cover,
+ logdir = LogDir,
+ config = SpecStartOpts#opts.config,
+ event_handlers = AllEvHs,
+ include = AllInclude,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT}}
+ end;
+ _ ->
+ {undefined,StartOpts}
+ end,
+ %% read config/userconfig from start flags
+ InitConfig = ct_config:prepare_config_list(Args),
+ case {TestSpec,Terms} of
+ {_,{error,_}=Error} ->
+ Error;
+ {[],_} ->
{error,no_testspec_specified};
- {value,{spec,Specs}} ->
- Relaxed = lists:keymember(allow_user_terms, 1, Args),
- %% using testspec as input for test
- case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
- {error,Reason} ->
- {error,Reason};
- TS ->
- {LogDir1,TSCoverFile,ConfigFiles1,EvHandlers1,Include1} =
- get_data_for_node(TS,node()),
- UserInclude =
- case application:get_env(common_test, include) of
- {ok,Include} -> Include++Include1;
- _ -> Include1
- end,
- application:set_env(common_test, include, UserInclude),
- LogDir2 = which_logdir(LogDir,LogDir1),
- CoverOpt = case {Cover,TSCoverFile} of
- {false,undef} -> [];
- {_,undef} -> [Cover];
- {false,_} -> [{cover,TSCoverFile}]
- end,
- case get_configfiles(ConfigFiles++ConfigFiles1,
- [], LogDir2,
- EvHandlers++EvHandlers1) of
- ok ->
- {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, CoverOpt, Args, LogDir2);
- Error ->
- Error
- end
+ {undefined,_} -> % no testspec used
+ case check_and_install_configfiles(InitConfig,
+ which(logdir,Opts#opts.logdir),
+ Opts#opts.event_handlers) of
+ ok -> % go on read tests from start flags
+ script_start3(Opts#opts{config=InitConfig}, Args);
+ Error ->
+ Error
end;
- false ->
- script_start3(false, ConfigFiles, EvHandlers, Args, LogDir, Cover)
+ {_,_} -> % testspec used
+ %% merge config from start flags with config from testspec
+ AllConfig = merge_vals([InitConfig, Opts#opts.config]),
+ case check_and_install_configfiles(AllConfig,
+ which(logdir,Opts#opts.logdir),
+ Opts#opts.event_handlers) of
+ ok -> % read tests from spec
+ {Run,Skip} = ct_testspec:prepare_tests(Terms, node()),
+ do_run(Run, Skip, Opts#opts{config=AllConfig}, Args);
+ Error ->
+ Error
+ end
end;
-script_start2(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
- script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover).
-script_start3(VtsOrShell, ConfigFiles, EvHandlers, Args, LogDir, Cover) ->
- case lists:keysearch(dir, 1, Args) of
- {value,{dir,[]}} ->
- {error,no_dir_specified};
- {value,{dir,Dirs}} ->
- script_start4(VtsOrShell, ConfigFiles, EvHandlers, tests(Dirs),
- Cover, Args, LogDir);
+script_start2(StartOpts, Args) ->
+ %% read config/userconfig from start flags
+ InitConfig = ct_config:prepare_config_list(Args),
+ case check_and_install_configfiles(InitConfig,
+ which(logdir,StartOpts#opts.logdir),
+ StartOpts#opts.event_handlers) of
+ ok -> % go on read tests from start flags
+ script_start3(StartOpts#opts{config=InitConfig}, Args);
+ Error ->
+ Error
+ end.
+
+check_and_install_configfiles(Configs, LogDir, EvHandlers) ->
+ case ct_config:check_config_files(Configs) of
false ->
- case lists:keysearch(suite, 1, Args) of
- {value,{suite,[]}} ->
+ install([{config,Configs},
+ {event_handler,EvHandlers}], LogDir);
+ {value,{error,{nofile,File}}} ->
+ {error,{cant_read_config_file,File}};
+ {value,{error,{wrong_config,Message}}}->
+ {error,{wrong_config,Message}};
+ {value,{error,{callback,Info}}} ->
+ {error,{cant_load_callback_module,Info}}
+ end.
+
+script_start3(StartOpts, Args) ->
+ case proplists:get_value(dir, Args) of
+ [] ->
+ {error,no_dir_specified};
+ Dirs when is_list(Dirs) ->
+ script_start4(StartOpts#opts{tests = tests(Dirs)}, Args);
+ undefined ->
+ case proplists:get_value(suite, Args) of
+ [] ->
{error,no_suite_specified};
- {value,{suite,Suites}} ->
- StepOrCover =
- case lists:keysearch(step, 1, Args) of
- {value,Step} -> Step;
- false -> Cover
- end,
- S2M = fun(S) ->
- {filename:dirname(S),
- list_to_atom(
- filename:rootname(filename:basename(S)))}
- end,
- DirMods = lists:map(S2M, Suites),
- {Specified,GroupsAndCases} =
- case {lists:keysearch(group, 1, Args),
- lists:keysearch('case', 1, Args)} of
- {{value,{_,Gs}},{value,{_,Cs}}} -> {true,Gs++Cs};
- {{value,{_,Gs}},_} -> {true,Gs};
- {_,{value,{_,Cs}}} -> {true,Cs};
- _ -> {false,[]}
- end,
- if Specified, length(GroupsAndCases) == 0 ->
- {error,no_case_or_group_specified};
- Specified, length(DirMods) > 1 ->
+ Suites when is_list(Suites) ->
+ StartOpts1 =
+ get_start_opt(step,
+ fun(Step) ->
+ StartOpts#opts{step = Step,
+ cover = undefined}
+ end, StartOpts, Args),
+ DirMods = [suite_to_test(S) || S <- Suites],
+ case groups_and_cases(proplists:get_value(group, Args),
+ proplists:get_value(testcase, Args)) of
+ Error = {error,_} ->
+ Error;
+ [] when DirMods =/= [] ->
+ Ts = tests(DirMods),
+ script_start4(StartOpts1#opts{tests = Ts}, Args);
+ GroupsAndCases when length(DirMods) == 1 ->
+ Ts = tests(DirMods, GroupsAndCases),
+ script_start4(StartOpts1#opts{tests = Ts}, Args);
+ [_,_|_] when length(DirMods) > 1 ->
{error,multiple_suites_and_cases};
- length(GroupsAndCases) > 0, length(DirMods) == 1 ->
- GsAndCs = lists:map(fun(C) -> list_to_atom(C) end,
- GroupsAndCases),
- script_start4(VtsOrShell, ConfigFiles, EvHandlers,
- tests(DirMods, GsAndCs),
- StepOrCover, Args, LogDir);
- not Specified, length(DirMods) > 0 ->
- script_start4(VtsOrShell, ConfigFiles, EvHandlers,
- tests(DirMods),
- StepOrCover, Args, LogDir);
- true ->
- {error,incorrect_suite_and_case_options}
+ _ ->
+ {error,incorrect_suite_option}
end;
- false when VtsOrShell=/=false ->
- script_start4(VtsOrShell, ConfigFiles, EvHandlers,
- [], Cover, Args, LogDir);
- false ->
- script_usage(),
- {error,incorrect_usage}
+ undefined ->
+ if StartOpts#opts.vts ; StartOpts#opts.shell ->
+ script_start4(StartOpts#opts{tests = []}, Args);
+ true ->
+ script_usage(),
+ {error,incorrect_usage}
+ end
end
end.
-script_start4(vts, ConfigFiles, EvHandlers, Tests, false, _Args, LogDir) ->
+script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers,
+ tests = Tests, logdir = LogDir}, _Args) ->
+ ConfigFiles =
+ lists:foldl(fun({ct_config_plain,CfgFiles}, AllFiles) when
+ is_list(hd(CfgFiles)) ->
+ AllFiles ++ CfgFiles;
+ ({ct_config_plain,CfgFile}, AllFiles) when
+ is_integer(hd(CfgFile)) ->
+ AllFiles ++ [CfgFile];
+ (_, AllFiles) ->
+ AllFiles
+ end, [], Config),
vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests);
-script_start4(shell, ConfigFiles, EvHandlers, _Tests, false, Args, LogDir) ->
- Opts = [{config,ConfigFiles},{event_handler,EvHandlers}],
- if ConfigFiles == [] ->
+
+script_start4(#opts{shell = true, config = Config, event_handlers = EvHandlers,
+ logdir = LogDir, testspecs = Specs}, _Args) ->
+ InstallOpts = [{config,Config},{event_handler,EvHandlers}],
+ if Config == [] ->
ok;
true ->
- io:format("\nInstalling: ~p\n\n", [ConfigFiles])
+ io:format("\nInstalling: ~p\n\n", [Config])
end,
- case install(Opts) of
+ case install(InstallOpts) of
ok ->
ct_util:start(interactive, LogDir),
- log_ts_names(Args),
+ log_ts_names(Specs),
io:nl(),
ok;
Error ->
Error
end;
-script_start4(vts, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) ->
- %% Add support later (maybe).
- script_usage(),
- erlang:halt();
-script_start4(shell, _CfgFs, _EvHs, _Tests, _Cover={cover,_}, _Args, _LogDir) ->
- %% Add support later (maybe).
- script_usage();
-script_start4(false, _CfgFs, _EvHs, Tests, Cover={cover,_}, Args, LogDir) ->
- do_run(Tests, [], [Cover], Args, LogDir);
-script_start4(false, _ConfigFiles, _EvHandlers, Tests, false, Args, LogDir) ->
- do_run(Tests, [], [], Args, LogDir);
-script_start4(false, _ConfigFiles, _EvHandlers, Test, Step, Args, LogDir) ->
- do_run(Test, [], [Step], Args, LogDir);
-script_start4(vts, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) ->
- script_usage(),
+
+script_start4(#opts{vts = true, cover = Cover}, _) ->
+ case Cover of
+ undefined ->
+ script_usage();
+ _ ->
+ %% Add support later (maybe).
+ io:format("\nCan't run cover in vts mode.\n\n", [])
+ end,
erlang:halt();
-script_start4(shell, _ConfigFiles, _EvHandlers, _Test, _Step, _Args, _LogDir) ->
- script_usage().
+
+script_start4(#opts{shell = true, cover = Cover}, _) ->
+ case Cover of
+ undefined ->
+ script_usage();
+ _ ->
+ %% Add support later (maybe).
+ io:format("\nCan't run cover in interactive mode.\n\n", [])
+ end;
+
+script_start4(Opts = #opts{tests = Tests}, Args) ->
+ do_run(Tests, [], Opts, Args).
%%%-----------------------------------------------------------------
%%% @spec script_usage() -> ok
-%%% @doc Print script usage information for <code>run_test</code>.
+%%% @doc Print usage information for <code>run_test</code>.
script_usage() ->
io:format("\n\nUsage:\n\n"),
io:format("Run tests in web based GUI:\n\n"
@@ -396,24 +478,29 @@ script_usage() ->
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
"\n\t[-dir TestDir1 TestDir2 .. TestDirN] |"
"\n\t[-suite Suite [-case Case]]"
- "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
+ "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
+ "\n\t[-multiply_timetraps N]"
+ "\n\t[-scale_timetraps]"
"\n\t[-basic_html]\n\n"),
io:format("Run tests from command line:\n\n"
"\trun_test [-dir TestDir1 TestDir2 .. TestDirN] |"
"\n\t[-suite Suite1 Suite2 .. SuiteN [-case Case1 Case2 .. CaseN]]"
"\n\t[-step [config | keep_inactive]]"
"\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
+ "\n\t[-userconfig CallbackModule ConfigFile1 .. ConfigFileN]"
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
"\n\t[-logdir LogDir]"
"\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
"\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
- "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
+ "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
- "\n\t[-basic_html]"
- "\n\t[-repeat N [-force_stop]] |"
+ "\n\t[-multiply_timetraps N]"
+ "\n\t[-scale_timetraps]"
+ "\n\t[-basic_html]"
+ "\n\t[-repeat N [-force_stop]] |"
"\n\t[-duration HHMMSS [-force_stop]] |"
"\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
io:format("Run tests using test specification:\n\n"
@@ -426,10 +513,12 @@ script_usage() ->
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
"\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
- "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
+ "\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
- "\n\t[-basic_html]"
- "\n\t[-repeat N [-force_stop]] |"
+ "\n\t[-multiply_timetraps N]"
+ "\n\t[-scale_timetraps]"
+ "\n\t[-basic_html]"
+ "\n\t[-repeat N [-force_stop]] |"
"\n\t[-duration HHMMSS [-force_stop]] |"
"\n\t[-until [YYMoMoDD]HHMMSS [-force_stop]]\n\n"),
io:format("Refresh the HTML index files:\n\n"
@@ -440,7 +529,6 @@ script_usage() ->
"\trun_test -shell"
"\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]\n\n").
-
%%%-----------------------------------------------------------------
%%% @hidden
@@ -468,7 +556,7 @@ install(Opts, LogDir) ->
[io:format(Fd, "~p.\n", [Opt]) || Opt <- Opts],
file:close(Fd),
ok;
- {error,Reason} ->
+ {error,Reason} ->
io:format("CT failed to install configuration data. Please "
"verify that the log directory exists and that "
"write permission is set.\n\n", []),
@@ -487,69 +575,58 @@ variables_file_name(Dir) ->
filename:join(Dir, "variables-"++atom_to_list(node())).
%%%-----------------------------------------------------------------
-%%% @hidden
+%%% @spec run_test(Opts) -> Result
+%%% Opts = [tuple()]
+%%% Result = [TestResult] | {error,Reason}
+%%%
+%%% @doc Start tests from the erlang shell or from an erlang program.
%%% @equiv ct:run_test/1
+%%%-----------------------------------------------------------------
-%% Opts = [OptTuples]
-%% OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
-%% {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} |
-%% {logdir,LogDir} | {cover,CoverSpecFile} | {step,StepOpts} |
-%% {silent_connections,Conns} | {event_handler,EventHandlers} |
-%% {include,InclDirs} | {auto_compile,Bool} |
-%% {repeat,N} | {duration,DurTime} | {until,StopTime} | {force_stop,Bool} |
-%% {decrypt,KeyOrFile}
-
-run_test(Opt) when is_tuple(Opt) ->
- run_test([Opt]);
-
-run_test(Opts) when is_list(Opts) ->
- case lists:keysearch(refresh_logs, 1, Opts) of
- {value,{_,RefreshDir}} ->
- refresh_logs(?abs(RefreshDir)),
- ok;
- false ->
- Tracing = start_trace(Opts),
+run_test(StartOpt) when is_tuple(StartOpt) ->
+ run_test([StartOpt]);
+
+run_test(StartOpts) when is_list(StartOpts) ->
+ case proplists:get_value(refresh_logs, StartOpts) of
+ undefined ->
+ Tracing = start_trace(StartOpts),
{ok,Cwd} = file:get_cwd(),
- io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
+ io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
Res =
- case ct_repeat:loop_test(func, Opts) of
+ case ct_repeat:loop_test(func, StartOpts) of
false ->
- case catch run_test1(Opts) of
- {'EXIT',Reason} ->
+ case catch run_test1(StartOpts) of
+ {'EXIT',Reason} ->
file:set_cwd(Cwd),
{error,Reason};
- Result ->
+ Result ->
Result
end;
Result ->
Result
end,
stop_trace(Tracing),
- Res
+ Res;
+ RefreshDir ->
+ refresh_logs(?abs(RefreshDir)),
+ ok
end.
-run_test1(Opts) ->
- LogDir =
- case lists:keysearch(logdir, 1, Opts) of
- {value,{_,LD}} when is_list(LD) -> LD;
- false -> "."
- end,
- CfgFiles =
- case lists:keysearch(config, 1, Opts) of
- {value,{_,Files=[File|_]}} when is_list(File) ->
- Files;
- {value,{_,File=[C|_]}} when is_integer(C) ->
- [File];
- {value,{_,[]}} ->
- [];
- false ->
- []
- end,
+run_test1(StartOpts) ->
+ %% logdir
+ LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end,
+ ".", StartOpts),
+ %% config & userconfig
+ CfgFiles = ct_config:get_config_file_list(StartOpts),
+
+ %% event handlers
EvHandlers =
- case lists:keysearch(event_handler, 1, Opts) of
- {value,{_,H}} when is_atom(H) ->
+ case proplists:get_value(event_handler, StartOpts) of
+ undefined ->
+ [];
+ H when is_atom(H) ->
[{H,[]}];
- {value,{_,H}} ->
+ H ->
Hs =
if is_tuple(H) -> [H];
is_list(H) -> H;
@@ -564,41 +641,39 @@ run_test1(Opts) ->
{EH,Args};
(_) ->
[]
- end, Hs));
- _ ->
- []
- end,
- SilentConns =
- case lists:keysearch(silent_connections, 1, Opts) of
- {value,{_,all}} ->
- [];
- {value,{_,Conns}} ->
- Conns;
- _ ->
- undefined
- end,
- Cover =
- case lists:keysearch(cover, 1, Opts) of
- {value,{_,CoverFile}} ->
- [{cover,?abs(CoverFile)}];
- _ ->
- []
+ end, Hs))
end,
+
+ %% silent connections
+ SilentConns = get_start_opt(silent_connections,
+ fun(all) -> [];
+ (Conns) -> Conns
+ end, StartOpts),
+ %% stylesheet
+ Stylesheet = get_start_opt(stylesheet,
+ fun(SS) -> ?abs(SS) end,
+ StartOpts),
+ %% code coverage
+ Cover = get_start_opt(cover,
+ fun(CoverFile) -> ?abs(CoverFile) end, StartOpts),
+
+ %% timetrap manipulation
+ MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts),
+ ScaleTT = get_start_opt(scale_timetraps, value, false, StartOpts),
+
+ %% auto compile & include files
Include =
- case lists:keysearch(auto_compile, 1, Opts) of
- {value,{auto_compile,ACBool}} ->
- application:set_env(common_test, auto_compile, ACBool),
- [];
- _ ->
+ case proplists:get_value(auto_compile, StartOpts) of
+ undefined ->
application:set_env(common_test, auto_compile, true),
InclDirs =
- case lists:keysearch(include, 1, Opts) of
- {value,{include,Incl}} when is_list(hd(Incl)) ->
- Incl;
- {value,{include,Incl}} when is_list(Incl) ->
- [Incl];
- false ->
- []
+ case proplists:get_value(include, StartOpts) of
+ undefined ->
+ [];
+ Incl when is_list(hd(Incl)) ->
+ Incl;
+ Incl when is_list(Incl) ->
+ [Incl]
end,
case os:getenv("CT_INCLUDE_PATH") of
false ->
@@ -609,117 +684,165 @@ run_test1(Opts) ->
AllInclDirs = InclDirs1++InclDirs,
application:set_env(common_test, include, AllInclDirs),
AllInclDirs
- end
+ end;
+ ACBool ->
+ application:set_env(common_test, auto_compile, ACBool),
+ []
end,
- case lists:keysearch(decrypt, 1, Opts) of
- {value,{_,Key={key,_}}} ->
+ %% decrypt config file
+ case proplists:get_value(decrypt, StartOpts) of
+ undefined ->
+ application:unset_env(common_test, decrypt);
+ Key={key,_} ->
application:set_env(common_test, decrypt, Key);
- {value,{_,{file,KeyFile}}} ->
- application:set_env(common_test, decrypt, {file,filename:absname(KeyFile)});
- false ->
- application:unset_env(common_test, decrypt)
+ {file,KeyFile} ->
+ application:set_env(common_test, decrypt, {file,?abs(KeyFile)})
end,
- case lists:keysearch(basic_html, 1, Opts) of
- {value,{basic_html,BasicHtmlBool}} ->
- application:set_env(common_test, basic_html, BasicHtmlBool);
- _ ->
- application:set_env(common_test, basic_html, false)
+ %% basic html - used by ct_logs
+ case proplists:get_value(basic_html, StartOpts) of
+ undefined ->
+ application:set_env(common_test, basic_html, false);
+ BasicHtmlBool ->
+ application:set_env(common_test, basic_html, BasicHtmlBool)
end,
- case lists:keysearch(spec, 1, Opts) of
- {value,{_,Specs}} ->
- Relaxed =
- case lists:keysearch(allow_user_terms, 1, Opts) of
- {value,{_,true}} -> true;
- _ -> false
- end,
+ %% stepped execution
+ Step = get_start_opt(step, value, StartOpts),
+
+ Opts = #opts{cover = Cover, step = Step, logdir = LogDir, config = CfgFiles,
+ event_handlers = EvHandlers, include = Include,
+ silent_connections = SilentConns,
+ stylesheet = Stylesheet,
+ multiply_timetraps = MultiplyTT,
+ scale_timetraps = ScaleTT},
+
+ %% test specification
+ case proplists:get_value(spec, StartOpts) of
+ undefined ->
+ case proplists:get_value(prepared_tests, StartOpts) of
+ undefined -> % use dir|suite|case
+ run_dir(Opts, StartOpts);
+ {{Run,Skip},Specs} -> % use prepared tests
+ run_prepared(Run, Skip, Opts#opts{testspecs = Specs}, StartOpts)
+ end;
+ Specs ->
+ Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts),
%% using testspec(s) as input for test
- run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover,
- replace_opt([{silent_connections,SilentConns}], Opts));
- false ->
- case lists:keysearch(prepared_tests, 1, Opts) of
- {value,{_,{Run,Skip},Specs}} -> % use prepared tests
- run_prepared(LogDir, CfgFiles, EvHandlers,
- Run, Skip, Cover,
- replace_opt([{silent_connections,SilentConns},
- {spec,Specs}],Opts));
- false -> % use dir|suite|case
- StepOrCover =
- case lists:keysearch(step, 1, Opts) of
- {value,Step} -> [Step];
- false -> Cover
- end,
- run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover,
- replace_opt([{silent_connections,SilentConns}], Opts))
- end
+ run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts)
end.
-replace_opt([O={Key,_Val}|Os], Opts) ->
- [O | replace_opt(Os, lists:keydelete(Key, 1, Opts))];
-replace_opt([], Opts) ->
- Opts.
-
-run_spec_file(LogDir, CfgFiles, EvHandlers, Include, Specs, Relaxed, Cover, Opts) ->
+run_spec_file(Relaxed,
+ Opts = #opts{testspecs = Specs, config = CfgFiles},
+ StartOpts) ->
Specs1 = case Specs of
[X|_] when is_integer(X) -> [Specs];
_ -> Specs
end,
- AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1),
+ AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1),
log_ts_names(AbsSpecs),
case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of
- {error,CTReason} ->
+ {Error,CTReason} when Error == error ; Error == 'EXIT' ->
exit(CTReason);
TS ->
- {LogDir1,TSCoverFile,CfgFiles1,EvHandlers1,Include1} =
- get_data_for_node(TS, node()),
- application:set_env(common_test, include, Include++Include1),
- LogDir2 = which_logdir(LogDir, LogDir1),
- CoverOpt = case {Cover,TSCoverFile} of
- {[],undef} -> [];
- {_,undef} -> Cover;
- {[],_} -> [{cover,TSCoverFile}]
- end,
- case get_configfiles(CfgFiles++CfgFiles1, [], LogDir2,
- EvHandlers++EvHandlers1) of
+ SpecOpts = get_data_for_node(TS, node()),
+ LogDir = choose_val(Opts#opts.logdir,
+ SpecOpts#opts.logdir),
+ AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]),
+ Cover = choose_val(Opts#opts.cover,
+ SpecOpts#opts.cover),
+ MultTT = choose_val(Opts#opts.multiply_timetraps,
+ SpecOpts#opts.multiply_timetraps),
+ ScaleTT = choose_val(Opts#opts.scale_timetraps,
+ SpecOpts#opts.scale_timetraps),
+ AllEvHs = merge_vals([Opts#opts.event_handlers,
+ SpecOpts#opts.event_handlers]),
+ AllInclude = merge_vals([Opts#opts.include,
+ SpecOpts#opts.include]),
+ application:set_env(common_test, include, AllInclude),
+
+ case check_and_install_configfiles(AllConfig,
+ which(logdir,LogDir),
+ AllEvHs) of
ok ->
+ Opts1 = Opts#opts{cover = Cover,
+ logdir = LogDir,
+ config = AllConfig,
+ event_handlers = AllEvHs,
+ include = AllInclude,
+ testspecs = AbsSpecs,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT},
{Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, CoverOpt,
- replace_opt([{spec,AbsSpecs}], Opts),
- LogDir2);
+ do_run(Run, Skip, Opts1, StartOpts);
{error,GCFReason} ->
exit(GCFReason)
end
end.
-run_prepared(LogDir, CfgFiles, EvHandlers, Run, Skip, Cover, Opts) ->
- case get_configfiles(CfgFiles, [], LogDir, EvHandlers) of
+run_prepared(Run, Skip, Opts = #opts{logdir = LogDir,
+ config = CfgFiles,
+ event_handlers = EvHandlers},
+ StartOpts) ->
+ case check_and_install_configfiles(CfgFiles, LogDir, EvHandlers) of
ok ->
- do_run(Run, Skip, Cover, Opts, LogDir);
+ do_run(Run, Skip, Opts, StartOpts);
{error,Reason} ->
exit(Reason)
- end.
-
-run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) ->
- AbsCfgFiles =
- lists:map(fun(F) ->
- AbsName = ?abs(F),
- case filelib:is_file(AbsName) of
- true -> AbsName;
- false -> exit({no_such_file,AbsName})
- end
- end, CfgFiles),
+ end.
+check_config_file(Callback, File)->
+ case code:is_loaded(Callback) of
+ false ->
+ case code:load_file(Callback) of
+ {module,_} -> ok;
+ {error,Why} -> exit({cant_load_callback_module,Why})
+ end;
+ _ ->
+ ok
+ end,
+ case Callback:check_parameter(File) of
+ {ok,{file,File}}->
+ ?abs(File);
+ {ok,{config,_}}->
+ File;
+ {error,{wrong_config,Message}}->
+ exit({wrong_config,{Callback,Message}});
+ {error,{nofile,File}}->
+ exit({no_such_file,?abs(File)})
+ end.
+
+run_dir(Opts = #opts{logdir = LogDir,
+ config = CfgFiles,
+ event_handlers = EvHandlers}, StartOpts) ->
+ AbsCfgFiles =
+ lists:map(fun({Callback,FileList})->
+ case code:is_loaded(Callback) of
+ {file,_Path}->
+ ok;
+ false ->
+ case code:load_file(Callback) of
+ {module,Callback}->
+ ok;
+ {error,_}->
+ exit({no_such_module,Callback})
+ end
+ end,
+ {Callback,
+ lists:map(fun(File)->
+ check_config_file(Callback, File)
+ end, FileList)}
+ end, CfgFiles),
case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir) of
ok -> ok;
{error,IReason} -> exit(IReason)
end,
- case lists:keysearch(dir,1,Opts) of
+ case lists:keysearch(dir, 1, StartOpts) of
{value,{_,Dirs=[Dir|_]}} when not is_integer(Dir),
length(Dirs)>1 ->
%% multiple dirs (no suite)
- do_run(tests(Dirs), [], StepOrCover, Opts, LogDir);
+ do_run(tests(Dirs), [], Opts, StartOpts);
false -> % no dir
%% fun for converting suite name to {Dir,Mod} tuple
S2M = fun(S) when is_list(S) ->
@@ -728,105 +851,120 @@ run_dir(LogDir, CfgFiles, EvHandlers, StepOrCover, Opts) ->
(A) ->
{".",A}
end,
- case lists:keysearch(suite, 1, Opts) of
+ case lists:keysearch(suite, 1, StartOpts) of
{value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) ->
{Dir,Mod} = S2M(Suite),
- case listify(proplists:get_value(group, Opts, [])) ++
- listify(proplists:get_value(testcase, Opts, [])) of
+ case listify(proplists:get_value(group, StartOpts, [])) ++
+ listify(proplists:get_value(testcase, StartOpts, [])) of
[] ->
- do_run(tests(Dir, listify(Mod)), [], StepOrCover, Opts, LogDir);
+ do_run(tests(Dir, listify(Mod)), [], Opts, StartOpts);
GsAndCs ->
- do_run(tests(Dir, Mod, GsAndCs), [], StepOrCover, Opts, LogDir)
+ do_run(tests(Dir, Mod, GsAndCs), [], Opts, StartOpts)
end;
{value,{_,Suites}} ->
- do_run(tests(lists:map(S2M, Suites)), [], StepOrCover, Opts, LogDir);
+ do_run(tests(lists:map(S2M, Suites)), [], Opts, StartOpts);
_ ->
exit(no_tests_specified)
- end;
+ end;
{value,{_,Dir}} ->
- case lists:keysearch(suite, 1, Opts) of
+ case lists:keysearch(suite, 1, StartOpts) of
{value,{_,Suite}} when is_integer(hd(Suite)) ; is_atom(Suite) ->
- Mod = if is_atom(Suite) -> Suite;
- true -> list_to_atom(Suite)
+ Mod = if is_atom(Suite) -> Suite;
+ true -> list_to_atom(Suite)
end,
- case listify(proplists:get_value(group, Opts, [])) ++
- listify(proplists:get_value(testcase, Opts, [])) of
+ case listify(proplists:get_value(group, StartOpts, [])) ++
+ listify(proplists:get_value(testcase, StartOpts, [])) of
[] ->
- do_run(tests(Dir, listify(Mod)), [], StepOrCover, Opts, LogDir);
+ do_run(tests(Dir, listify(Mod)), [], Opts, StartOpts);
GsAndCs ->
- do_run(tests(Dir, Mod, GsAndCs), [], StepOrCover, Opts, LogDir)
+ do_run(tests(Dir, Mod, GsAndCs), [], Opts, StartOpts)
end;
{value,{_,Suites=[Suite|_]}} when is_list(Suite) ->
Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites),
- do_run(tests(delistify(Dir), Mods), [], StepOrCover, Opts, LogDir);
+ do_run(tests(delistify(Dir), Mods), [], Opts, StartOpts);
{value,{_,Suites}} ->
- do_run(tests(delistify(Dir), Suites), [], StepOrCover, Opts, LogDir);
+ do_run(tests(delistify(Dir), Suites), [], Opts, StartOpts);
false -> % no suite, only dir
- do_run(tests(listify(Dir)), [], StepOrCover, Opts, LogDir)
- end
+ do_run(tests(listify(Dir)), [], Opts, StartOpts)
+ end
end.
%%%-----------------------------------------------------------------
-%%% @hidden
+%%% @spec run_testspec(TestSpec) -> Result
+%%% TestSpec = [term()]
%%%
+%%% @doc Run test specified by <code>TestSpec</code>. The terms are
+%%% the same as those used in test specification files.
+%%% @equiv ct:run_testspec/1
+%%%-----------------------------------------------------------------
-%% using testspec(s) as input for test
run_testspec(TestSpec) ->
{ok,Cwd} = file:get_cwd(),
io:format("~nCommon Test starting (cwd is ~s)~n~n", [Cwd]),
case catch run_testspec1(TestSpec) of
- {'EXIT',Reason} ->
+ {'EXIT',Reason} ->
file:set_cwd(Cwd),
{error,Reason};
- Result ->
+ Result ->
Result
end.
run_testspec1(TestSpec) ->
- case ct_testspec:collect_tests_from_list(TestSpec,false) of
- {error,CTReason} ->
+ case catch ct_testspec:collect_tests_from_list(TestSpec, false) of
+ {E,CTReason} when E == error ; E == 'EXIT' ->
exit(CTReason);
TS ->
- {LogDir,TSCoverFile,CfgFiles,EvHandlers,Include} =
- get_data_for_node(TS,node()),
- case os:getenv("CT_INCLUDE_PATH") of
- false ->
- application:set_env(common_test, include, Include);
- CtInclPath ->
- EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]),
- application:set_env(common_test, include, EnvInclude++Include)
- end,
- CoverOpt = if TSCoverFile == undef -> [];
- true -> [{cover,TSCoverFile}]
- end,
- case get_configfiles(CfgFiles,[],LogDir,EvHandlers) of
+ Opts = get_data_for_node(TS, node()),
+
+ AllInclude =
+ case os:getenv("CT_INCLUDE_PATH") of
+ false ->
+ Opts#opts.include;
+ CtInclPath ->
+ EnvInclude = string:tokens(CtInclPath, [$:,$ ,$,]),
+ EnvInclude++Opts#opts.include
+ end,
+ application:set_env(common_test, include, AllInclude),
+
+ case check_and_install_configfiles(Opts#opts.config,
+ which(logdir,Opts#opts.logdir),
+ Opts#opts.event_handlers) of
ok ->
- {Run,Skip} = ct_testspec:prepare_tests(TS,node()),
- do_run(Run,Skip,CoverOpt,[],LogDir);
+ Opts1 = Opts#opts{testspecs = [TestSpec],
+ include = AllInclude},
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ do_run(Run, Skip, Opts1, []);
{error,GCFReason} ->
exit(GCFReason)
end
end.
-
get_data_for_node(#testspec{logdir=LogDirs,
cover=CoverFs,
config=Cfgs,
+ userconfig=UsrCfgs,
event_handler=EvHs,
- include=Incl}, Node) ->
- LogDir = case lists:keysearch(Node,1,LogDirs) of
- {value,{Node,Dir}} -> Dir;
- false -> "."
+ include=Incl,
+ multiply_timetraps=MTs,
+ scale_timetraps=STs}, Node) ->
+ LogDir = case proplists:get_value(Node, LogDirs) of
+ undefined -> ".";
+ Dir -> Dir
end,
- Cover = case lists:keysearch(Node,1,CoverFs) of
- {value,{Node,CovFile}} -> CovFile;
- false -> undef
- end,
- ConfigFiles = [F || {N,F} <- Cfgs, N==Node],
+ Cover = proplists:get_value(Node, CoverFs),
+ MT = proplists:get_value(Node, MTs),
+ ST = proplists:get_value(Node, STs),
+ ConfigFiles = [{?ct_config_txt,F} || {N,F} <- Cfgs, N==Node] ++
+ [CBF || {N,CBF} <- UsrCfgs, N==Node],
EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node],
Include = [I || {N,I} <- Incl, N==Node],
- {LogDir,Cover,ConfigFiles,EvHandlers,Include}.
-
+ #opts{logdir = LogDir,
+ cover = Cover,
+ config = ConfigFiles,
+ event_handlers = EvHandlers,
+ include = Include,
+ multiply_timetraps = MT,
+ scale_timetraps = ST}.
refresh_logs(LogDir) ->
{ok,Cwd} = file:get_cwd(),
@@ -851,11 +989,27 @@ refresh_logs(LogDir) ->
end
end.
-which_logdir(".",Dir) ->
+which(logdir, undefined) ->
+ ".";
+which(logdir, Dir) ->
Dir;
-which_logdir(Dir,_) ->
- Dir.
-
+which(multiply_timetraps, undefined) ->
+ 1;
+which(multiply_timetraps, MT) ->
+ MT;
+which(scale_timetraps, undefined) ->
+ false;
+which(scale_timetraps, ST) ->
+ ST.
+
+choose_val(undefined, V1) ->
+ V1;
+choose_val(V0, _V1) ->
+ V0.
+
+merge_vals(Vs) ->
+ lists:append(Vs).
+
listify([C|_]=Str) when is_integer(C) -> [Str];
listify(L) when is_list(L) -> L;
listify(E) -> [E].
@@ -885,6 +1039,22 @@ run(TestDirs) ->
install([]),
do_run(tests(TestDirs), []).
+suite_to_test(Suite) ->
+ {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}.
+
+groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and
+ ((Cs == undefined) or (Cs == [])) ->
+ [];
+groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] ->
+ [list_to_atom(C) || C <- Cs];
+groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] ->
+ [{list_to_atom(G),all} || G <- Gs];
+groups_and_cases([G], Cs) ->
+ [{list_to_atom(G),[list_to_atom(C) || C <- Cs]}];
+groups_and_cases([_,_|_] , Cs) when Cs =/= [] ->
+ {error,multiple_groups_and_cases};
+groups_and_cases(_Gs, _Cs) ->
+ {error,incorrect_group_or_case_option}.
tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) ->
[{?testdir(TestDir,Suites),ensure_atom(Suites),all}];
@@ -901,30 +1071,42 @@ tests(TestDir) when is_list(TestDir), is_integer(hd(TestDir)) ->
tests(TestDirs) when is_list(TestDirs), is_list(hd(TestDirs)) ->
[{?testdir(TestDir,all),all,all} || TestDir <- TestDirs].
-do_run(Tests, Opt) ->
- do_run(Tests, [], Opt, [], ".").
+do_run(Tests, Misc) when is_list(Misc) ->
+ do_run(Tests, Misc, ".").
-do_run(Tests, Opt, LogDir) ->
- do_run(Tests, [], Opt, [], LogDir).
+do_run(Tests, Misc, LogDir) when is_list(Misc) ->
+ Opts =
+ case proplists:get_value(step, Misc) of
+ undefined ->
+ #opts{};
+ StepOpts ->
+ #opts{step = StepOpts}
+ end,
+ Opts1 =
+ case proplists:get_value(cover, Misc) of
+ undefined ->
+ Opts;
+ CoverFile ->
+ Opts#opts{cover = CoverFile}
+ end,
+ do_run(Tests, [], Opts1#opts{logdir = LogDir}, []).
-do_run(Tests, Skip, Opt, Args, LogDir) ->
+do_run(Tests, Skip, Opts, Args) ->
+ #opts{cover = Cover} = Opts,
case code:which(test_server) of
non_existing ->
exit({error,no_path_to_test_server});
_ ->
- Opt1 =
- case lists:keysearch(cover, 1, Opt) of
- {value,{_,CoverFile}} ->
- case ct_cover:get_spec(CoverFile) of
- {error,Reason} ->
- exit({error,Reason});
- Spec ->
- [{cover_spec,Spec} |
- lists:keydelete(cover, 1, Opt)]
- end;
- _ ->
- Opt
- end,
+ Opts1 = if Cover == undefined ->
+ Opts;
+ true ->
+ case ct_cover:get_spec(Cover) of
+ {error,Reason} ->
+ exit({error,Reason});
+ CoverSpec ->
+ Opts#opts{coverspec = CoverSpec}
+ end
+ end,
%% This env variable is used by test_server to determine
%% which framework it runs under.
case os:getenv("TEST_SERVER_FRAMEWORK") of
@@ -935,50 +1117,39 @@ do_run(Tests, Skip, Opt, Args, LogDir) ->
Other ->
erlang:display(list_to_atom("Note: TEST_SERVER_FRAMEWORK = " ++ Other))
end,
- case ct_util:start(LogDir) of
+ case ct_util:start(Opts#opts.logdir) of
{error,interactive_mode} ->
io:format("CT is started in interactive mode. "
"To exit this mode, run ct:stop_interactive().\n"
"To enter the interactive mode again, "
"run ct:start_interactive()\n\n",[]),
{error,interactive_mode};
-
_Pid ->
- %% save style sheet info
- case lists:keysearch(stylesheet, 1, Args) of
- {value,{_,SSFile}} ->
- ct_util:set_testdata({stylesheet,SSFile});
- _ ->
- ct_util:set_testdata({stylesheet,undefined})
- end,
-
- case lists:keysearch(silent_connections, 1, Args) of
- {value,{silent_connections,undefined}} ->
- ok;
- {value,{silent_connections,[]}} ->
+ %% save stylesheet info
+ ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}),
+ %% enable silent connections
+ case Opts#opts.silent_connections of
+ [] ->
Conns = ct_util:override_silence_all_connections(),
ct_logs:log("Silent connections", "~p", [Conns]);
- {value,{silent_connections,Cs}} ->
- Conns = lists:map(fun(S) when is_list(S) ->
- list_to_atom(S);
- (A) -> A
- end, Cs),
+ Conns when is_list(Conns) ->
ct_util:override_silence_connections(Conns),
ct_logs:log("Silent connections", "~p", [Conns]);
_ ->
ok
end,
- log_ts_names(Args),
+ log_ts_names(Opts1#opts.testspecs),
TestSuites = suite_tuples(Tests),
- {SuiteMakeErrors,AllMakeErrors} =
+ {_TestSuites1,SuiteMakeErrors,AllMakeErrors} =
case application:get_env(common_test, auto_compile) of
{ok,false} ->
- SuitesNotFound = verify_suites(TestSuites),
- {SuitesNotFound,SuitesNotFound};
+ {TestSuites1,SuitesNotFound} =
+ verify_suites(TestSuites),
+ {TestSuites1,SuitesNotFound,SuitesNotFound};
_ ->
{SuiteErrs,HelpErrs} = auto_compile(TestSuites),
- {SuiteErrs,SuiteErrs++HelpErrs}
+ {TestSuites,SuiteErrs,SuiteErrs++HelpErrs}
end,
case continue(AllMakeErrors) of
@@ -986,7 +1157,7 @@ do_run(Tests, Skip, Opt, Args, LogDir) ->
SavedErrors = save_make_errors(SuiteMakeErrors),
ct_repeat:log_loop_info(Args),
{Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors),
- R = do_run_test(Tests1, Skip1, Opt1),
+ R = do_run_test(Tests1, Skip1, Opts1),
ct_util:stop(normal),
R;
false ->
@@ -1012,7 +1183,7 @@ auto_compile(TestSuites) ->
case application:get_env(common_test, include) of
{ok,UserInclDirs} when length(UserInclDirs) > 0 ->
io:format("Including the following directories:~n"),
- [begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end ||
+ [begin io:format("~p~n",[UserInclDir]), {i,UserInclDir} end ||
UserInclDir <- UserInclDirs];
_ ->
[]
@@ -1020,11 +1191,11 @@ auto_compile(TestSuites) ->
SuiteMakeErrors =
lists:flatmap(fun({TestDir,Suite} = TS) ->
case run_make(suites, TestDir, Suite, UserInclude) of
- {error,{make_failed,Bad}} ->
+ {error,{make_failed,Bad}} ->
[{TS,Bad}];
- {error,_} ->
+ {error,_} ->
[{TS,[filename:join(TestDir,"*_SUITE")]}];
- _ ->
+ _ ->
[]
end
end, TestSuites),
@@ -1048,39 +1219,63 @@ auto_compile(TestSuites) ->
true -> % already visited
{Done,Failed}
end
- end, {[],[]}, TestSuites),
+ end, {[],[]}, TestSuites),
{SuiteMakeErrors,lists:reverse(HelpMakeErrors)}.
%% verify that specified test suites exist (if auto compile is disabled)
verify_suites(TestSuites) ->
io:nl(),
- Verify =
- fun({Dir,Suite},NotFound) ->
+ Verify =
+ fun({Dir,Suite}=DS,{Found,NotFound}) ->
case locate_test_dir(Dir, Suite) of
{ok,TestDir} ->
if Suite == all ->
- NotFound;
+ {[DS|Found],NotFound};
true ->
- Beam = filename:join(TestDir, atom_to_list(Suite)++".beam"),
+ Beam = filename:join(TestDir,
+ atom_to_list(Suite)++".beam"),
case filelib:is_regular(Beam) of
- true ->
- NotFound;
- false ->
- Name = filename:join(TestDir, atom_to_list(Suite)),
- io:format("Suite ~w not found in directory ~s~n",
- [Suite,TestDir]),
- [{{Dir,Suite},[Name]} | NotFound]
+ true ->
+ {[DS|Found],NotFound};
+ false ->
+ case code:is_loaded(Suite) of
+ {file,SuiteFile} ->
+ %% test suite is already loaded and
+ %% since auto_compile == false,
+ %% let's assume the user has
+ %% loaded the beam file explicitly
+ ActualDir = filename:dirname(SuiteFile),
+ {[{ActualDir,Suite}|Found],NotFound};
+ false ->
+ Name =
+ filename:join(TestDir,
+ atom_to_list(Suite)),
+ io:format(user,
+ "Suite ~w not found"
+ "in directory ~s~n",
+ [Suite,TestDir]),
+ {Found,[{DS,[Name]}|NotFound]}
+ end
end
end;
{error,_Reason} ->
- io:format("Directory ~s is invalid~n", [Dir]),
- Name = filename:join(Dir, atom_to_list(Suite)),
- [{{Dir,Suite},[Name]} | NotFound]
+ case code:is_loaded(Suite) of
+ {file,SuiteFile} ->
+ %% test suite is already loaded and since
+ %% auto_compile == false, let's assume the
+ %% user has loaded the beam file explicitly
+ ActualDir = filename:dirname(SuiteFile),
+ {[{ActualDir,Suite}|Found],NotFound};
+ false ->
+ io:format(user, "Directory ~s is invalid~n", [Dir]),
+ Name = filename:join(Dir, atom_to_list(Suite)),
+ {Found,[{DS,[Name]}|NotFound]}
+ end
end
end,
- lists:reverse(lists:foldl(Verify, [], TestSuites)).
-
-
+ {ActualFound,Missing} = lists:foldl(Verify, {[],[]}, TestSuites),
+ {lists:reverse(ActualFound),lists:reverse(Missing)}.
+
save_make_errors([]) ->
[];
save_make_errors(Errors) ->
@@ -1096,7 +1291,7 @@ get_bad_suites([{{_TestDir,_Suite},Failed}|Errors], BadSuites) ->
get_bad_suites([], BadSuites) ->
BadSuites.
-
+
%%%-----------------------------------------------------------------
%%% @hidden
@@ -1107,7 +1302,7 @@ step(TestDir, Suite, Case) ->
%%%-----------------------------------------------------------------
%%% @hidden
%%% @equiv ct:step/4
-step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case),
+step(TestDir, Suite, Case, Opts) when is_list(TestDir), is_atom(Suite), is_atom(Case),
Suite =/= all, Case =/= all ->
do_run([{TestDir,Suite,Case}], [{step,Opts}]).
@@ -1140,7 +1335,7 @@ final_tests([{TestDir,Suites,_}|Tests],
Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites,
S == S1, TD == TestDir],
- Final1 = [{TestDir,S,all} || S <- Suites],
+ Final1 = [{TestDir,S,all} || S <- Suites],
final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad);
final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) ->
@@ -1173,7 +1368,7 @@ final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) ->
final_tests([], Final, Skip, _Bad) ->
{lists:reverse(Final),Skip}.
-continue([]) ->
+continue([]) ->
true;
continue(_MakeErrors) ->
io:nl(),
@@ -1214,7 +1409,7 @@ set_group_leader_same_as_shell() ->
false
end
end,
- case [P || P <- processes(), GS2or3(P),
+ case [P || P <- processes(), GS2or3(P),
true == lists:keymember(shell,1,element(2,process_info(P,dictionary)))] of
[GL|_] ->
group_leader(GL, self());
@@ -1238,29 +1433,29 @@ check_and_add([{TestDir0,M,_} | Tests], Added) ->
check_and_add([], _) ->
ok.
-do_run_test(Tests, Skip, Opt) ->
+do_run_test(Tests, Skip, Opts) ->
case check_and_add(Tests, []) of
ok ->
ct_util:set_testdata({stats,{0,0,{0,0}}}),
ct_util:set_testdata({cover,undefined}),
test_server_ctrl:start_link(local),
- case lists:keysearch(cover_spec, 1, Opt) of
- {value,{_,CovData={CovFile,
- CovNodes,
- _CovImport,
- CovExport,
- #cover{app = CovApp,
- level = CovLevel,
- excl_mods = CovExcl,
- incl_mods = CovIncl,
- cross = CovCross,
- src = _CovSrc}}}} ->
+ case Opts#opts.coverspec of
+ CovData={CovFile,
+ CovNodes,
+ _CovImport,
+ CovExport,
+ #cover{app = CovApp,
+ level = CovLevel,
+ excl_mods = CovExcl,
+ incl_mods = CovIncl,
+ cross = CovCross,
+ src = _CovSrc}} ->
ct_logs:log("COVER INFO","Using cover specification file: ~s~n"
"App: ~w~n"
"Cross cover: ~w~n"
"Including ~w modules~n"
"Excluding ~w modules",
- [CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]),
+ [CovFile,CovApp,CovCross,length(CovIncl),length(CovExcl)]),
%% cover export file will be used for export and import
%% between tests so make sure it doesn't exist initially
@@ -1293,33 +1488,37 @@ do_run_test(Tests, Skip, Opt) ->
true;
_ ->
false
- end,
+ end,
%% let test_server expand the test tuples and count no of cases
{Suites,NoOfCases} = count_test_cases(Tests, Skip),
Suites1 = delete_dups(Suites),
NoOfTests = length(Tests),
NoOfSuites = length(Suites1),
- ct_util:warn_duplicates(Suites1),
+ ct_util:warn_duplicates(Suites1),
{ok,Cwd} = file:get_cwd(),
io:format("~nCWD set to: ~p~n", [Cwd]),
if NoOfCases == unknown ->
- io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n",
+ io:format("~nTEST INFO: ~w test(s), ~w suite(s)~n~n",
[NoOfTests,NoOfSuites]),
- ct_logs:log("TEST INFO","~w test(s), ~w suite(s)",
+ ct_logs:log("TEST INFO","~w test(s), ~w suite(s)",
[NoOfTests,NoOfSuites]);
true ->
- io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n",
+ io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n",
[NoOfTests,NoOfCases,NoOfSuites]),
- ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)",
+ ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)",
[NoOfTests,NoOfCases,NoOfSuites])
end,
+
+ test_server_ctrl:multiply_timetraps(Opts#opts.multiply_timetraps),
+ test_server_ctrl:scale_timetraps(Opts#opts.scale_timetraps),
+
ct_event:notify(#event{name=start_info,
node=node(),
data={NoOfTests,NoOfSuites,NoOfCases}}),
- CleanUp = add_jobs(Tests, Skip, Opt, []),
+ CleanUp = add_jobs(Tests, Skip, Opts, []),
unlink(whereis(test_server_ctrl)),
- catch test_server_ctrl:wait_finish(),
- %% check if last testcase has left a "dead" trace window
+ catch test_server_ctrl:wait_finish(),
+ %% check if last testcase has left a "dead" trace window
%% behind, and if so, kill it
case ct_util:get_testdata(interpret) of
{_What,kill,{TCPid,AttPid}} ->
@@ -1327,8 +1526,8 @@ do_run_test(Tests, Skip, Opt) ->
_ ->
ok
end,
- lists:foreach(fun(Suite) ->
- maybe_cleanup_interpret(Suite, Opt)
+ lists:foreach(fun(Suite) ->
+ maybe_cleanup_interpret(Suite, Opts#opts.step)
end, CleanUp);
Error ->
Error
@@ -1344,9 +1543,9 @@ count_test_cases(Tests, Skip) ->
SendResult = fun(Me, Result) -> Me ! {no_of_cases,Result} end,
TSPid = test_server_ctrl:start_get_totals(SendResult),
Ref = erlang:monitor(process, TSPid),
- add_jobs(Tests, Skip, [], []),
+ add_jobs(Tests, Skip, #opts{}, []),
{Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref),
- erlang:demonitor(Ref),
+ erlang:demonitor(Ref, [flush]),
test_server_ctrl:stop_get_totals(),
{Suites,NoOfCases}.
@@ -1354,11 +1553,11 @@ count_test_cases1(0, N, Suites, _) ->
{lists:flatten(Suites), N};
count_test_cases1(Jobs, N, Suites, Ref) ->
receive
- {no_of_cases,{Ss,N1}} ->
+ {no_of_cases,{Ss,N1}} ->
count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref);
- {'DOWN', Ref, _, _, _} ->
+ {'DOWN', Ref, _, _, _} ->
{[],0}
- end.
+ end.
add_known(unknown, _) ->
unknown;
@@ -1367,72 +1566,86 @@ add_known(_, unknown) ->
add_known(N, N1) ->
N+N1.
-add_jobs([{TestDir,all,_}|Tests], Skip, Opt, CleanUp) ->
+add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) ->
Name = get_name(TestDir),
case catch test_server_ctrl:add_dir_with_skip(Name, TestDir,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, CleanUp)
+ add_jobs(Tests, Skip, Opts, CleanUp)
end;
-add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opt, CleanUp) when is_atom(Suite) ->
- add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp);
-add_jobs([{TestDir,Suites,all}|Tests], Skip, Opt, CleanUp) when is_list(Suites) ->
+add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) ->
+ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp);
+add_jobs([{TestDir,Suites,all}|Tests], Skip, Opts, CleanUp) when is_list(Suites) ->
Name = get_name(TestDir) ++ ".suites",
case catch test_server_ctrl:add_module_with_skip(Name, Suites,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, CleanUp)
+ add_jobs(Tests, Skip, Opts, CleanUp)
end;
-add_jobs([{TestDir,Suite,all}|Tests], Skip, Opt, CleanUp) ->
- case maybe_interpret(Suite, all, Opt) of
+add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) ->
+ case maybe_interpret(Suite, all, Opts) of
ok ->
Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite),
case catch test_server_ctrl:add_module_with_skip(Name, [Suite],
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
+ add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
end;
Error ->
Error
end;
-add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opt, CleanUp) when is_atom(Case) ->
- add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp);
-add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opt, CleanUp) when is_list(Cases) ->
- case maybe_interpret(Suite, Cases, Opt) of
+
+%% group
+add_jobs([{TestDir,Suite,[{GroupName,_Cases}]}|Tests], Skip, Opts, CleanUp) when
+ is_atom(GroupName) ->
+ add_jobs([{TestDir,Suite,GroupName}|Tests], Skip, Opts, CleanUp);
+add_jobs([{TestDir,Suite,{GroupName,_Cases}}|Tests], Skip, Opts, CleanUp) when
+ is_atom(GroupName) ->
+ add_jobs([{TestDir,Suite,GroupName}|Tests], Skip, Opts, CleanUp);
+
+%% test case
+add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) ->
+ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp);
+
+add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opts, CleanUp) when is_list(Cases) ->
+ Cases1 = lists:map(fun({GroupName,_}) when is_atom(GroupName) -> GroupName;
+ (Case) -> Case
+ end, Cases),
+ case maybe_interpret(Suite, Cases1, Opts) of
ok ->
Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases",
- case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases,
+ case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases1,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
+ add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
end;
Error ->
Error
end;
-add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opt, CleanUp) when is_atom(Case) ->
- case maybe_interpret(Suite, Case, Opt) of
+add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) ->
+ case maybe_interpret(Suite, Case, Opts) of
ok ->
- Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++
+ Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++
atom_to_list(Case),
case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case,
skiplist(TestDir,Skip)) of
- {'EXIT',_} ->
+ {'EXIT',_} ->
CleanUp;
_ ->
wait_for_idle(),
- add_jobs(Tests, Skip, Opt, [Suite|CleanUp])
+ add_jobs(Tests, Skip, Opts, [Suite|CleanUp])
end;
Error ->
Error
@@ -1453,7 +1666,7 @@ wait_for_idle() ->
idle -> ok;
{'DOWN', Ref, _, _, _} -> error
end,
- erlang:demonitor(Ref),
+ erlang:demonitor(Ref, [flush]),
ct_util:update_last_run_index(),
Result
end.
@@ -1482,7 +1695,7 @@ get_name(Dir) ->
end,
Base = filename:basename(TestDir),
case filename:basename(filename:dirname(TestDir)) of
- "" ->
+ "" ->
Base;
TopDir ->
TopDir ++ "." ++ Base
@@ -1513,15 +1726,15 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
{i,CtInclude},
{i,XmerlInclude},
debug_info],
- Result =
+ Result =
if Mod == all ; Targets == helpmods ->
case (catch ct_make:all([noexec|ErlFlags])) of
- {'EXIT',_} = Failure ->
+ {'EXIT',_} = Failure ->
Failure;
MakeInfo ->
FileTest = fun(F, suites) -> is_suite(F);
- (F, helpmods) -> not is_suite(F);
- (_, _) -> true end,
+ (F, helpmods) -> not is_suite(F)
+ end,
Files = lists:flatmap(fun({F,out_of_date}) ->
case FileTest(F, Targets) of
true -> [F];
@@ -1535,7 +1748,7 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
true ->
(catch ct_make:files([Mod], [load|ErlFlags]))
end,
-
+
ok = file:set_cwd(Cwd),
%% send finished_make notification
ct_event:notify(#event{name=finished_make,
@@ -1549,7 +1762,7 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
{error,{make_crashed,TestDir,Reason}};
{error,ModInfo} ->
io:format("{error,make_failed}\n", []),
- Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo,
+ Bad = [filename:join(TestDir, M) || {M,R} <- ModInfo,
R == error],
{error,{make_failed,Bad}}
end;
@@ -1561,8 +1774,8 @@ run_make(Targets, TestDir0, Mod, UserInclude) ->
get_dir(App, Dir) ->
filename:join(code:lib_dir(App), Dir).
-maybe_interpret(Suite, Cases, [{step,StepOpts}]) ->
- %% if other suite has run before this one, check if last testcase
+maybe_interpret(Suite, Cases, #opts{step = StepOpts}) when StepOpts =/= undefined ->
+ %% if other suite has run before this one, check if last testcase
%% has left a "dead" trace window behind, and if so, kill it
case ct_util:get_testdata(interpret) of
{_What,kill,{TCPid,AttPid}} ->
@@ -1605,7 +1818,7 @@ maybe_interpret2(Suite, Cases, StepOpts) ->
WinOp = case lists:member(keep_inactive, ensure_atom(StepOpts)) of
true -> no_kill;
false -> kill
- end,
+ end,
ct_util:set_testdata({interpret,{{Suite,Cases},WinOp,
{undefined,undefined}}}),
ok.
@@ -1621,37 +1834,44 @@ set_break_on_config(Suite, StepOpts) ->
ok
end.
-maybe_cleanup_interpret(Suite, [{step,_}]) ->
- i:iq(Suite);
-maybe_cleanup_interpret(_, _) ->
- ok.
+maybe_cleanup_interpret(_, undefined) ->
+ ok;
+maybe_cleanup_interpret(Suite, _) ->
+ i:iq(Suite).
+
+log_ts_names([]) ->
+ ok;
+log_ts_names(Specs) ->
+ List = lists:map(fun(Name) ->
+ Name ++ " "
+ end, Specs),
+ ct_logs:log("Test Specification file(s)", "~s",
+ [lists:flatten(List)]).
-log_ts_names(Args) ->
- case lists:keysearch(spec, 1, Args) of
- {value,{_,Specs}} ->
- List = lists:map(fun(Name) ->
- Name ++ " "
- end, Specs),
- ct_logs:log("Test Specification file(s)", "~s",
- [lists:flatten(List)]);
- _ ->
- ok
- end.
-
merge_arguments(Args) ->
merge_arguments(Args, []).
merge_arguments([LogDir={logdir,_}|Args], Merged) ->
merge_arguments(Args, handle_arg(replace, LogDir, Merged));
+
merge_arguments([CoverFile={cover,_}|Args], Merged) ->
merge_arguments(Args, handle_arg(replace, CoverFile, Merged));
-merge_arguments([Arg={_,_}|Args], Merged) ->
+
+merge_arguments([{'case',TC}|Args], Merged) ->
+ merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged));
+
+merge_arguments([Arg|Args], Merged) ->
merge_arguments(Args, handle_arg(merge, Arg, Merged));
+
merge_arguments([], Merged) ->
Merged.
handle_arg(replace, {Key,Elems}, [{Key,_}|Merged]) ->
[{Key,Elems}|Merged];
+handle_arg(merge, {event_handler_init,Elems}, [{event_handler_init,PrevElems}|Merged]) ->
+ [{event_handler_init,PrevElems++["add"|Elems]}|Merged];
+handle_arg(merge, {userconfig,Elems}, [{userconfig,PrevElems}|Merged]) ->
+ [{userconfig,PrevElems++["add"|Elems]}|Merged];
handle_arg(merge, {Key,Elems}, [{Key,PrevElems}|Merged]) ->
[{Key,PrevElems++Elems}|Merged];
handle_arg(Op, Arg, [Other|Merged]) ->
@@ -1659,6 +1879,164 @@ handle_arg(Op, Arg, [Other|Merged]) ->
handle_arg(_,Arg,[]) ->
[Arg].
+get_start_opt(Key, IfExists, Args) ->
+ get_start_opt(Key, IfExists, undefined, Args).
+
+get_start_opt(Key, IfExists, IfNotExists, Args) ->
+ case lists:keysearch(Key, 1, Args) of
+ {value,{Key,Val}} when is_function(IfExists) ->
+ IfExists(Val);
+ {value,{Key,Val}} when IfExists == value ->
+ Val;
+ {value,{Key,_Val}} ->
+ IfExists;
+ _ when is_function(IfNotExists) ->
+ IfNotExists();
+ _ ->
+ IfNotExists
+ end.
+
+event_handler_args2opts(Args) ->
+ case proplists:get_value(event_handler, Args) of
+ undefined ->
+ event_handler_args2opts([], Args);
+ EHs ->
+ event_handler_args2opts([{list_to_atom(EH),[]} || EH <- EHs], Args)
+ end.
+event_handler_args2opts(Default, Args) ->
+ case proplists:get_value(event_handler_init, Args) of
+ undefined ->
+ Default;
+ EHs ->
+ event_handler_init_args2opts(EHs)
+ end.
+event_handler_init_args2opts([EH, Arg, "and" | EHs]) ->
+ [{list_to_atom(EH),lists:flatten(io_lib:format("~s",[Arg]))} |
+ event_handler_init_args2opts(EHs)];
+event_handler_init_args2opts([EH, Arg]) ->
+ [{list_to_atom(EH),lists:flatten(io_lib:format("~s",[Arg]))}];
+event_handler_init_args2opts([]) ->
+ [].
+
+%% This function reads pa and pz arguments, converts dirs from relative
+%% to absolute, and re-inserts them in the code path. The order of the
+%% dirs in the code path remain the same. Note however that since this
+%% function is only used for arguments "pre run_test erl_args", the order
+%% relative dirs "post run_test erl_args" is not kept!
+rel_to_abs(CtArgs) ->
+ {PA,PZ} = get_pa_pz(CtArgs, [], []),
+ io:format(user, "~n", []),
+ [begin
+ code:del_path(filename:basename(D)),
+ Abs = filename:absname(D),
+ code:add_pathz(Abs),
+ if D /= Abs ->
+ io:format(user, "Converting ~p to ~p and re-inserting "
+ "with add_pathz/1~n",
+ [D, Abs]);
+ true ->
+ ok
+ end
+ end || D <- PZ],
+ [begin
+ code:del_path(filename:basename(D)),
+ Abs = filename:absname(D),
+ code:add_patha(Abs),
+ if D /= Abs ->
+ io:format(user, "Converting ~p to ~p and re-inserting "
+ "with add_patha/1~n",
+ [D, Abs]);
+ true ->ok
+ end
+ end || D <- PA],
+ io:format(user, "~n", []).
+
+get_pa_pz([{pa,Dirs} | Args], PA, PZ) ->
+ get_pa_pz(Args, PA ++ Dirs, PZ);
+get_pa_pz([{pz,Dirs} | Args], PA, PZ) ->
+ get_pa_pz(Args, PA, PZ ++ Dirs);
+get_pa_pz([_ | Args], PA, PZ) ->
+ get_pa_pz(Args, PA, PZ);
+get_pa_pz([], PA, PZ) ->
+ {PA,PZ}.
+
+%% This function translates ct:run_test/1 start options
+%% to run_test start arguments (on the init arguments format) -
+%% this is useful mainly for testing the ct_run start functions.
+opts2args(EnvStartOpts) ->
+ lists:flatmap(fun({config,CfgFiles}) ->
+ [{ct_config,[CfgFiles]}];
+ ({userconfig,{CBM,CfgStr=[X|_]}}) when is_integer(X) ->
+ [{userconfig,[atom_to_list(CBM),CfgStr]}];
+ ({userconfig,{CBM,CfgStrs}}) when is_list(CfgStrs) ->
+ [{userconfig,[atom_to_list(CBM) | CfgStrs]}];
+ ({userconfig,UserCfg}) when is_list(UserCfg) ->
+ Strs =
+ lists:map(fun({CBM,CfgStr=[X|_]}) when is_integer(X) ->
+ [atom_to_list(CBM),CfgStr,"and"];
+ ({CBM,CfgStrs}) when is_list(CfgStrs) ->
+ [atom_to_list(CBM) | CfgStrs] ++ ["and"]
+ end, UserCfg),
+ [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)),
+ [{userconfig,lists:reverse(StrsR)}];
+ ({testcase,Case}) when is_atom(Case) ->
+ [{'case',[atom_to_list(Case)]}];
+ ({testcase,Cases}) ->
+ [{'case',[atom_to_list(C) || C <- Cases]}];
+ ({'case',Cases}) ->
+ [{'case',[atom_to_list(C) || C <- Cases]}];
+ ({allow_user_terms,true}) ->
+ [{allow_user_terms,[]}];
+ ({allow_user_terms,false}) ->
+ [];
+ ({auto_compile,false}) ->
+ [{no_auto_compile,[]}];
+ ({auto_compile,true}) ->
+ [];
+ ({scale_timetraps,true}) ->
+ [{scale_timetraps,[]}];
+ ({scale_timetraps,false}) ->
+ [];
+ ({force_stop,true}) ->
+ [{force_stop,[]}];
+ ({force_stop,false}) ->
+ [];
+ ({decrypt,{key,Key}}) ->
+ [{ct_decrypt_key,[Key]}];
+ ({decrypt,{file,File}}) ->
+ [{ct_decrypt_file,[File]}];
+ ({basic_html,true}) ->
+ ({basic_html,[]});
+ ({basic_html,false}) ->
+ [];
+ ({event_handler,EH}) when is_atom(EH) ->
+ [{event_handler,[atom_to_list(EH)]}];
+ ({event_handler,EHs}) when is_list(EHs) ->
+ [{event_handler,[atom_to_list(EH) || EH <- EHs]}];
+ ({event_handler,{EH,Arg}}) when is_atom(EH) ->
+ ArgStr = lists:flatten(io_lib:format("~p", [Arg])),
+ [{event_handler_init,[atom_to_list(EH),ArgStr]}];
+ ({event_handler,{EHs,Arg}}) when is_list(EHs) ->
+ ArgStr = lists:flatten(io_lib:format("~p", [Arg])),
+ Strs = lists:map(fun(EH) ->
+ [atom_to_list(EH),ArgStr,"and"]
+ end, EHs),
+ [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)),
+ [{event_handler_init,lists:reverse(StrsR)}];
+ ({Opt,As=[A|_]}) when is_atom(A) ->
+ [{Opt,[atom_to_list(Atom) || Atom <- As]}];
+ ({Opt,Strs=[S|_]}) when is_list(S) ->
+ [{Opt,Strs}];
+ ({Opt,A}) when is_atom(A) ->
+ [{Opt,[atom_to_list(A)]}];
+ ({Opt,I}) when is_integer(I) ->
+ [{Opt,[integer_to_list(I)]}];
+ ({Opt,S}) when is_list(S) ->
+ [{Opt,[S]}];
+ (Opt) ->
+ Opt
+ end, EnvStartOpts).
+
locate_test_dir(Dir, Suite) ->
TestDir = case ct_util:is_test_dir(Dir) of
true -> Dir;
@@ -1723,18 +2101,18 @@ start_trace(Args) ->
case file:consult(TraceSpec) of
{ok,Terms} ->
case catch do_trace(Terms) of
- ok ->
+ ok ->
true;
{_,Error} ->
io:format("Warning! Tracing not started. Reason: ~p~n~n",
[Error]),
false
- end;
+ end;
{_,Error} ->
io:format("Warning! Tracing not started. Reason: ~p~n~n",
[Error]),
false
- end;
+ end;
false ->
false
end.
@@ -1746,61 +2124,22 @@ do_trace(Terms) ->
case dbg:tpl(M,[{'_',[],[{return_trace}]}]) of
{error,What} -> exit({error,{tracing_failed,What}});
_ -> ok
- end;
+ end;
({f,M,F}) ->
case dbg:tpl(M,F,[{'_',[],[{return_trace}]}]) of
{error,What} -> exit({error,{tracing_failed,What}});
_ -> ok
- end;
+ end;
(Huh) ->
exit({error,{unrecognized_trace_term,Huh}})
end, Terms),
ok.
-
+
stop_trace(true) ->
dbg:stop_clear();
stop_trace(false) ->
ok.
-preload() ->
- io:format("~nLoading Common Test and Test Server modules...~n~n"),
- preload_mod([ct_logs,
- ct_make,
- ct_telnet,
- ct,
- ct_master,
- ct_testspec,
- ct_cover,
- ct_master_event,
- ct_util,
- ct_event,
- ct_master_logs,
- ct_framework,
- teln,
- ct_ftp,
- ct_rpc,
- unix_telnet,
- ct_gen_conn,
- ct_line,
- ct_snmp,
- test_server_sup,
- test_server,
- test_server_ctrl,
- test_server_h,
- test_server_line,
- test_server_node]).
-
-preload_mod([M|Ms]) ->
- case code:is_loaded(M) of
- false ->
- {module,M} = code:load_file(M),
- preload_mod(Ms);
- _ ->
- ok
- end;
-preload_mod([]) ->
- ok.
-
ensure_atom(Atom) when is_atom(Atom) ->
Atom;
ensure_atom(String) when is_list(String), is_integer(hd(String)) ->
@@ -1809,4 +2148,3 @@ ensure_atom(List) when is_list(List) ->
[ensure_atom(Item) || Item <- List];
ensure_atom(Other) ->
Other.
-