This module implements the command line interface for running
%%% tests and some basic functions for common test case issues
%%% such as configuration and logging.
%%%
%%% Run this function once before first test.
%%%
%%% From this mode all test case support functions can be executed
%%% directly from the erlang shell. The interactive mode can also be
%%% started from the OS command line with ct_run -shell
%%% [-config File...].
%%%
%%% If any functions using "required config data" (e.g. telnet or
%%% ftp functions) are to be called from the erlang shell, config data
%%% must first be required with ct:require/2.
%%%
%%% Example: Require one node with a telnet connection and an
%%% ftp connection. Name the node a:
%%%
This function returns the matching value(s) or config element(s),
%%% given a config variable key or its associated name
%%% (if one has been specified with require/2 or a
%%% require statement).
%%%
%%% If a config variable is defined in multiple files and you want to
%%% access all possible values, use the all option. The
%%% values will be returned in a list and the order of the elements
%%% corresponds to the order that the config files were specified at
%%% startup.
%%%
%%% If you want config elements (key-value tuples) returned as result
%%% instead of values, use the element option.
%%% The returned elements will then be on the form {Required,Value}
%%%
%%% @see get_config/1
%%% @see get_config/2
%%% @see require/1
%%% @see require/2
get_config(Required,Default,Opts) ->
ct_config:get_config(Required,Default,Opts).
%%%-----------------------------------------------------------------
%%% @spec reload_config(Required) -> ValueOrElement
%%% Required = KeyOrName | {KeyOrName,SubKey} | {KeyOrName,SubKey,SubKey}
%%% KeyOrName = atom()
%%% SubKey = atom()
%%% ValueOrElement = term()
%%%
%%% @doc Reload config file which contains specified configuration key.
%%%
%%% This function performs updating of the configuration data from which the
%%% given configuration variable was read, and returns the (possibly) new
%%% value of this variable.
%%% Note that if some variables were present in the configuration but are not loaded
%%% using this function, they will be removed from the configuration table together
%%% with their aliases.
%%%
reload_config(Required)->
ct_config:reload_config(Required).
%%%-----------------------------------------------------------------
%%% @spec get_testspec_terms() -> TestSpecTerms | undefined
%%% TestSpecTerms = [{Tag,Value}]
%%% Value = [term()]
%%%
%%% @doc Get a list of all test specification terms used to
%%% configure and run this test.
%%%
get_testspec_terms() ->
case ct_util:get_testdata(testspec) of
undefined ->
undefined;
CurrSpecRec ->
ct_testspec:testspec_rec2list(CurrSpecRec)
end.
%%%-----------------------------------------------------------------
%%% @spec get_testspec_terms(Tags) -> TestSpecTerms | undefined
%%% Tags = [Tag] | Tag
%%% Tag = atom()
%%% TestSpecTerms = [{Tag,Value}] | {Tag,Value}
%%% Value = [{Node,term()}] | [term()]
%%% Node = atom()
%%%
%%% @doc Read one or more terms from the test specification used
%%% to configure and run this test. Tag is any valid test specification
%%% tag, such as e.g. This function is meant for printing a string directly from a
%%% test case to the test case log file.
%%%
%%% This function is meant for printing a string from a test case
%%% to the console.
%%%
%%% This function is meant for printing a string from a test case,
%%% both to the test case log file and to the console.
%%%
%%% .*"),
lists:flatmap(fun(Str) ->
case re:run(Str, MP) of
{match,_} -> [];
nomatch -> [Str]
end
end, Strs);
capture_get([]) ->
test_server:capture_get().
%%%-----------------------------------------------------------------
%%% @spec fail(Reason) -> ok
%%% Reason = term()
%%%
%%% @doc Terminate a test case with the given error
%%%
Reason.
fail(Reason) ->
try
exit({test_case_failed,Reason})
catch
Class:R ->
case erlang:get_stacktrace() of
[{?MODULE,fail,1,_}|Stk] -> ok;
Stk -> ok
end,
erlang:raise(Class, R, Stk)
end.
%%%-----------------------------------------------------------------
%%% @spec fail(Format, Args) -> ok
%%% Format = string()
%%% Args = list()
%%%
%%% @doc Terminate a test case with an error message specified
%%% by a format string and a list of values (used as arguments to
%%%
io_lib:format/2).
fail(Format, Args) ->
try io_lib:format(Format, Args) of
Str ->
try
exit({test_case_failed,lists:flatten(Str)})
catch
Class:R ->
case erlang:get_stacktrace() of
[{?MODULE,fail,2,_}|Stk] -> ok;
Stk -> ok
end,
erlang:raise(Class, R, Stk)
end
catch
_:BadArgs ->
exit({BadArgs,{?MODULE,fail,[Format,Args]}})
end.
%%%-----------------------------------------------------------------
%%% @spec comment(Comment) -> ok
%%% Comment = term()
%%%
%%% @doc Print the given
Comment in the comment field in
%%% the table on the test suite result page.
%%%
%%%
If called several times, only the last comment is printed.
%%% The test case return value {comment,Comment}
%%% overwrites the string set by this function.
comment(Comment) when is_list(Comment) ->
Formatted =
case (catch io_lib:format("~ts",[Comment])) of
{'EXIT',_} -> % it's a list not a string
io_lib:format("~p",[Comment]);
String ->
String
end,
send_html_comment(lists:flatten(Formatted));
comment(Comment) ->
Formatted = io_lib:format("~p",[Comment]),
send_html_comment(lists:flatten(Formatted)).
%%%-----------------------------------------------------------------
%%% @spec comment(Format, Args) -> ok
%%% Format = string()
%%% Args = list()
%%%
%%% @doc Print the formatted string in the comment field in
%%% the table on the test suite result page.
%%%
%%%
The Format and Args arguments are
%%% used in call to io_lib:format/2 in order to create
%%% the comment string. The behaviour of comment/2 is
%%% otherwise the same as the comment/1 function (see
%%% above for details).
comment(Format, Args) when is_list(Format), is_list(Args) ->
Formatted =
case (catch io_lib:format(Format, Args)) of
{'EXIT',Reason} -> % bad args
exit({Reason,{?MODULE,comment,[Format,Args]}});
String ->
lists:flatten(String)
end,
send_html_comment(Formatted).
send_html_comment(Comment) ->
Html = "
" ++ Comment ++ "",
ct_util:set_testdata({{comment,group_leader()},Html}),
test_server:comment(Html).
%%%-----------------------------------------------------------------
%%% @spec make_priv_dir() -> ok | {error,Reason}
%%% Reason = term()
%%% @doc If the test has been started with the create_priv_dir
%%% option set to manual_per_tc, in order for the test case to use
%%% the private directory, it must first create it by calling
%%% this function.
make_priv_dir() ->
test_server:make_priv_dir().
%%%-----------------------------------------------------------------
%%% @spec get_target_name(Handle) -> {ok,TargetName} | {error,Reason}
%%% Handle = handle()
%%% TargetName = target_name()
%%% @doc Return the name of the target that the given connection
%%% belongs to.
get_target_name(Handle) ->
ct_util:get_target_name(Handle).
%%%-----------------------------------------------------------------
%%% @spec parse_table(Data) -> {Heading,Table}
%%% Data = [string()]
%%% Heading = tuple()
%%% Table = [tuple()]
%%%
%%% @doc Parse the printout from an SQL table and return a list of tuples.
%%%
%%%
The printout to parse would typically be the result of a
%%% select command in SQL. The returned
%%% Table is a list of tuples, where each tuple is a row
%%% in the table.
%%%
%%%
Heading is a tuple of strings representing the
%%% headings of each column in the table.
parse_table(Data) ->
ct_util:parse_table(Data).
%%%-----------------------------------------------------------------
%%% @spec listenv(Telnet) -> [Env]
%%% Telnet = term()
%%% Env = {Key,Value}
%%% Key = string()
%%% Value = string()
%%%
%%% @doc Performs the listenv command on the given telnet connection
%%% and returns the result as a list of Key-Value pairs.
listenv(Telnet) ->
ct_util:listenv(Telnet).
%%%-----------------------------------------------------------------
%%% @spec testcases(TestDir, Suite) -> Testcases | {error,Reason}
%%% TestDir = string()
%%% Suite = atom()
%%% Testcases = list()
%%% Reason = term()
%%%
%%% @doc Returns all test cases in the specified suite.
testcases(TestDir, Suite) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
case (catch Suite:all()) of
{'EXIT',Reason} ->
{error,Reason};
TCs ->
TCs
end
end.
make_and_load(Dir, Suite) ->
EnvInclude =
case os:getenv("CT_INCLUDE_PATH") of
false -> [];
CtInclPath -> string:tokens(CtInclPath, [$:,$ ,$,])
end,
StartInclude =
case init:get_argument(include) of
{ok,[Dirs]} -> Dirs;
_ -> []
end,
UserInclude = EnvInclude ++ StartInclude,
case ct_run:run_make(Dir, Suite, UserInclude) of
MErr = {error,_} ->
MErr;
_ ->
TestDir = ct_util:get_testdir(Dir, Suite),
File = filename:join(TestDir, atom_to_list(Suite)),
case code:soft_purge(Suite) of
true ->
code:load_abs(File);
false -> % will use loaded
{module,Suite}
end
end.
%%%-----------------------------------------------------------------
%%% @spec userdata(TestDir, Suite) -> SuiteUserData | {error,Reason}
%%% TestDir = string()
%%% Suite = atom()
%%% SuiteUserData = [term()]
%%% Reason = term()
%%%
%%% @doc Returns any data specified with the tag
userdata
%%% in the list of tuples returned from
Suite:suite/0.
userdata(TestDir, Suite) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
Info = (catch Suite:suite()),
get_userdata(Info, "suite/0")
end.
get_userdata({'EXIT',{Undef,_}}, Spec) when Undef == undef;
Undef == function_clause ->
{error,list_to_atom(Spec ++ " is not defined")};
get_userdata({'EXIT',Reason}, Spec) ->
{error,{list_to_atom("error in " ++ Spec),Reason}};
get_userdata(List, _) when is_list(List) ->
Fun = fun({userdata,Data}, Acc) -> [Data | Acc];
(_, Acc) -> Acc
end,
case lists:foldl(Fun, [], List) of
Terms ->
lists:flatten(lists:reverse(Terms))
end;
get_userdata(_BadTerm, Spec) ->
{error,list_to_atom(Spec ++ " must return a list")}.
%%%-----------------------------------------------------------------
%%% @spec userdata(TestDir, Suite, GroupOrCase) -> TCUserData | {error,Reason}
%%% TestDir = string()
%%% Suite = atom()
%%% GroupOrCase = {group,GroupName} | atom()
%%% GroupName = atom()
%%% TCUserData = [term()]
%%% Reason = term()
%%%
%%% @doc Returns any data specified with the tag
userdata
%%% in the list of tuples returned from
Suite:group(GroupName)
%%% or
Suite:Case().
userdata(TestDir, Suite, {group,GroupName}) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
Info = (catch apply(Suite, group, [GroupName])),
get_userdata(Info, "group("++atom_to_list(GroupName)++")")
end;
userdata(TestDir, Suite, Case) when is_atom(Case) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
Info = (catch apply(Suite, Case, [])),
get_userdata(Info, atom_to_list(Case)++"/0")
end.
%%%-----------------------------------------------------------------
%%% @spec get_status() -> TestStatus | {error,Reason} | no_tests_running
%%% TestStatus = [StatusElem]
%%% StatusElem = {current,TestCaseInfo} | {successful,Successful} |
%%% {failed,Failed} | {skipped,Skipped} | {total,Total}
%%% TestCaseInfo = {Suite,TestCase} | [{Suite,TestCase}]
%%% Suite = atom()
%%% TestCase = atom()
%%% Successful = integer()
%%% Failed = integer()
%%% Skipped = {UserSkipped,AutoSkipped}
%%% UserSkipped = integer()
%%% AutoSkipped = integer()
%%% Total = integer()
%%% Reason = term()
%%%
%%% @doc Returns status of ongoing test. The returned list contains info about
%%% which test case is currently executing (a list of cases when a
%%% parallel test case group is executing), as well as counters for
%%% successful, failed, skipped, and total test cases so far.
get_status() ->
case get_testdata(curr_tc) of
{ok,TestCase} ->
case get_testdata(stats) of
{ok,{Ok,Failed,Skipped={UserSkipped,AutoSkipped}}} ->
[{current,TestCase},
{successful,Ok},
{failed,Failed},
{skipped,Skipped},
{total,Ok+Failed+UserSkipped+AutoSkipped}];
Err1 -> Err1
end;
Err2 -> Err2
end.
get_testdata(Key) ->
case catch ct_util:get_testdata(Key) of
{error,ct_util_server_not_running} ->
no_tests_running;
Error = {error,_Reason} ->
Error;
{'EXIT',_Reason} ->
no_tests_running;
undefined ->
{error,no_testdata};
[CurrTC] when Key == curr_tc ->
{ok,CurrTC};
Data ->
{ok,Data}
end.
%%%-----------------------------------------------------------------
%%% @spec abort_current_testcase(Reason) -> ok | {error,ErrorReason}
%%% Reason = term()
%%% ErrorReason = no_testcase_running | parallel_group
%%%
%%% @doc
When calling this function, the currently executing test case will be aborted.
%%% It is the user's responsibility to know for sure which test case is currently
%%% executing. The function is therefore only safe to call from a function which
%%% has been called (or synchronously invoked) by the test case.
%%%
%%%
Reason, the reason for aborting the test case, is printed
%%% in the test case log.
abort_current_testcase(Reason) ->
test_server_ctrl:abort_current_testcase(Reason).
%%%-----------------------------------------------------------------
%%% @spec get_event_mgr_ref() -> EvMgrRef
%%% EvMgrRef = atom()
%%%
%%% @doc
Call this function in order to get a reference to the
%%% CT event manager. The reference can be used to e.g. add
%%% a user specific event handler while tests are running.
%%% Example:
%%% gen_event:add_handler(ct:get_event_mgr_ref(), my_ev_h, [])
get_event_mgr_ref() ->
?CT_EVMGR_REF.
%%%-----------------------------------------------------------------
%%% @spec encrypt_config_file(SrcFileName, EncryptFileName) ->
%%% ok | {error,Reason}
%%% SrcFileName = string()
%%% EncryptFileName = string()
%%% Reason = term()
%%%
%%% @doc
This function encrypts the source config file with DES3 and
%%% saves the result in file EncryptFileName. The key,
%%% a string, must be available in a text file named
%%% .ct_config.crypt in the current directory, or the
%%% home directory of the user (it is searched for in that order).
%%%
See the Common Test User's Guide for information about using
%%% encrypted config files when running tests.
%%%
See the crypto application for details on DES3
%%% encryption/decryption.
encrypt_config_file(SrcFileName, EncryptFileName) ->
ct_config:encrypt_config_file(SrcFileName, EncryptFileName).
%%%-----------------------------------------------------------------
%%% @spec encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) ->
%%% ok | {error,Reason}
%%% SrcFileName = string()
%%% EncryptFileName = string()
%%% KeyOrFile = {key,string()} | {file,string()}
%%% Reason = term()
%%%
%%% @doc
This function encrypts the source config file with DES3 and
%%% saves the result in the target file EncryptFileName.
%%% The encryption key to use is either the value in
%%% {key,Key} or the value stored in the file specified
%%% by {file,File}.
%%%
See the Common Test User's Guide for information about using
%%% encrypted config files when running tests.
%%%
See the crypto application for details on DES3
%%% encryption/decryption.
encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) ->
ct_config:encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile).
%%%-----------------------------------------------------------------
%%% @spec decrypt_config_file(EncryptFileName, TargetFileName) ->
%%% ok | {error,Reason}
%%% EncryptFileName = string()
%%% TargetFileName = string()
%%% Reason = term()
%%%
%%% @doc
This function decrypts EncryptFileName, previously
%%% generated with encrypt_config_file/2/3. The original
%%% file contents is saved in the target file. The encryption key, a
%%% string, must be available in a text file named
%%% .ct_config.crypt in the current directory, or the
%%% home directory of the user (it is searched for in that order).
decrypt_config_file(EncryptFileName, TargetFileName) ->
ct_config:decrypt_config_file(EncryptFileName, TargetFileName).
%%%-----------------------------------------------------------------
%%% @spec decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) ->
%%% ok | {error,Reason}
%%% EncryptFileName = string()
%%% TargetFileName = string()
%%% KeyOrFile = {key,string()} | {file,string()}
%%% Reason = term()
%%%
%%% @doc
This function decrypts EncryptFileName, previously
%%% generated with encrypt_config_file/2/3. The original
%%% file contents is saved in the target file. The key must have the
%%% the same value as that used for encryption.
decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) ->
ct_config:decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile).
%%%-----------------------------------------------------------------
%%% @spec add_config(Callback, Config) -> ok | {error, Reason}
%%% Callback = atom()
%%% Config = string()
%%% Reason = term()
%%%
%%% @doc
This function loads configuration variables using the
%%% given callback module and configuration string. Callback module
%%% should be either loaded or present in the code part. Loaded
%%% configuration variables can later be removed using
%%% remove_config/2 function.
add_config(Callback, Config)->
ct_config:add_config(Callback, Config).
%%%-----------------------------------------------------------------
%%% @spec remove_config(Callback, Config) -> ok
%%% Callback = atom()
%%% Config = string()
%%% Reason = term()
%%%
%%% @doc
This function removes configuration variables (together with
%%% their aliases) which were loaded with specified callback module and
%%% configuration string.
remove_config(Callback, Config) ->
ct_config:remove_config(Callback, Config).
%%%-----------------------------------------------------------------
%%% @spec timetrap(Time) -> ok
%%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity | Func
%%% Hours = integer()
%%% Mins = integer()
%%% Secs = integer()
%%% Millisecs = integer() | float()
%%% Func = {M,F,A} | fun()
%%% M = atom()
%%% F = atom()
%%% A = list()
%%%
%%% @doc
Use this function to set a new timetrap for the running test case.
%%% If the argument is Func, the timetrap will be triggered
%%% when this function returns. Func may also return a new
%%% Time value, which in that case will be the value for the
%%% new timetrap.
timetrap(Time) ->
test_server:timetrap_cancel(),
test_server:timetrap(Time).
%%%-----------------------------------------------------------------
%%% @spec get_timetrap_info() -> {Time,Scale}
%%% Time = integer() | infinity
%%% Scale = true | false
%%%
%%% @doc
Read info about the timetrap set for the current test case.
%%% Scale indicates if Common Test will attempt to automatically
%%% compensate timetraps for runtime delays introduced by e.g. tools like
%%% cover.
get_timetrap_info() ->
test_server:get_timetrap_info().
%%%-----------------------------------------------------------------
%%% @spec sleep(Time) -> ok
%%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity
%%% Hours = integer()
%%% Mins = integer()
%%% Secs = integer()
%%% Millisecs = integer() | float()
%%%
%%% @doc
This function, similar to timer:sleep/1, suspends the test
%%% case for specified time. However, this function also multiplies
%%% Time with the 'multiply_timetraps' value (if set) and under
%%% certain circumstances also scales up the time automatically
%%% if 'scale_timetraps' is set to true (default is false).
sleep({hours,Hs}) ->
sleep(trunc(Hs * 1000 * 60 * 60));
sleep({minutes,Ms}) ->
sleep(trunc(Ms * 1000 * 60));
sleep({seconds,Ss}) ->
sleep(trunc(Ss * 1000));
sleep(Time) ->
test_server:adjusted_sleep(Time).
%%%-----------------------------------------------------------------
%%% @spec notify(Name,Data) -> ok
%%% Name = atom()
%%% Data = term()
%%%
%%% @doc
Sends a asynchronous notification of type Name with
%%% Datato the common_test event manager. This can later be
%%% caught by any installed event manager.
%%% @see //stdlib/gen_event
notify(Name,Data) ->
ct_event:notify(Name, Data).
%%%-----------------------------------------------------------------
%%% @spec sync_notify(Name,Data) -> ok
%%% Name = atom()
%%% Data = term()
%%%
%%% @doc
Sends a synchronous notification of type Name with
%%% Datato the common_test event manager. This can later be
%%% caught by any installed event manager.
%%% @see //stdlib/gen_event
sync_notify(Name,Data) ->
ct_event:sync_notify(Name, Data).
%%%-----------------------------------------------------------------
%%% @spec break(Comment) -> ok | {error,Reason}
%%% Comment = string()
%%% Reason = {multiple_cases_running,TestCases} |
%%% 'enable break with release_shell option'
%%% TestCases = [atom()]
%%%
%%% @doc
This function will cancel any active timetrap and pause the
%%% execution of the current test case until the user calls the
%%% continue/0 function. It gives the user the opportunity
%%% to interact with the erlang node running the tests, e.g. for
%%% debugging purposes or for manually executing a part of the
%%% test case. If a parallel group is executing, break/2
%%% should be called instead.
%%%
A cancelled timetrap will not be automatically
%%% reactivated after the break, but must be started exlicitly with
%%% ct:timetrap/1
%%%
In order for the break/continue functionality to work,
%%% Common Test must release the shell process controlling stdin.
%%% This is done by setting the release_shell start option
%%% to true. See the User's Guide for more information.
break(Comment) ->
case {ct_util:get_testdata(starter),
ct_util:get_testdata(release_shell)} of
{ct,ReleaseSh} when ReleaseSh /= true ->
Warning = "ct:break/1 can only be used if release_shell == true.\n",
ct_logs:log("Warning!", Warning, []),
io:format(user, "Warning! " ++ Warning, []),
{error,'enable break with release_shell option'};
_ ->
case get_testdata(curr_tc) of
{ok,{_,_TestCase}} ->
test_server:break(?MODULE, Comment);
{ok,Cases} when is_list(Cases) ->
{error,{'multiple cases running',
[TC || {_,TC} <- Cases]}};
Error = {error,_} ->
Error;
Error ->
{error,Error}
end
end.
%%%-----------------------------------------------------------------
%%% @spec break(TestCase, Comment) -> ok | {error,Reason}
%%% TestCase = atom()
%%% Comment = string()
%%% Reason = 'test case not running' |
%%% 'enable break with release_shell option'
%%%
%%% @doc
This function works the same way as break/1,
%%% only the TestCase argument makes it possible to
%%% pause a test case executing in a parallel group. The
%%% continue/1 function should be used to resume
%%% execution of TestCase.
%%%
See break/1 for more details.
break(TestCase, Comment) ->
case {ct_util:get_testdata(starter),
ct_util:get_testdata(release_shell)} of
{ct,ReleaseSh} when ReleaseSh /= true ->
Warning = "ct:break/2 can only be used if release_shell == true.\n",
ct_logs:log("Warning!", Warning, []),
io:format(user, "Warning! " ++ Warning, []),
{error,'enable break with release_shell option'};
_ ->
case get_testdata(curr_tc) of
{ok,Cases} when is_list(Cases) ->
case lists:keymember(TestCase, 2, Cases) of
true ->
test_server:break(?MODULE, TestCase, Comment);
false ->
{error,'test case not running'}
end;
{ok,{_,TestCase}} ->
test_server:break(?MODULE, TestCase, Comment);
Error = {error,_} ->
Error;
Error ->
{error,Error}
end
end.
%%%-----------------------------------------------------------------
%%% @spec continue() -> ok
%%%
%%% @doc
This function must be called in order to continue after a
%%% test case (not executing in a parallel group) has called
%%% break/1.
continue() ->
test_server:continue().
%%%-----------------------------------------------------------------
%%% @spec continue(TestCase) -> ok
%%% TestCase = atom()
%%%
%%% @doc
This function must be called in order to continue after a
%%% test case has called break/2. If the paused test case,
%%% TestCase, executes in a parallel group, this
%%% function - rather than continue/0 - must be used
%%% in order to let the test case proceed.
continue(TestCase) ->
test_server:continue(TestCase).