This module implements the command line interface for running %%% tests and some basic functions for common test case issues %%% such as configuration and logging.
%%% %%%Test Suite Support Macros
%%% %%%The config macro is defined in ct.hrl. This
%%% macro should be used to retrieve information from the
%%% Config variable sent to all test cases. It is used with two
%%% arguments, where the first is the name of the configuration
%%% variable you wish to retrieve, and the second is the Config
%%% variable supplied to the test case.
Possible configuration variables include:
%%%data_dir - Data file directory.priv_dir - Scratch file directory.init_per_suite/1 or
%%% init_per_testcase/2 in the test suite.ct:require/2 is called,
%%% e.g. ct:require(mynodename,{node,[telnet]})
%%%
%%% @type target_name() = var_name(). The name of a target.
%%%
%%% @type handle() = ct_gen_conn:handle() | term(). The identity of a
%%% specific connection.
-module(ct).
-include("ct.hrl").
%% Command line user interface for running tests
-export([install/1, run/1, run/2, run/3,
run_test/1, run_testspec/1, step/3, step/4,
start_interactive/0, stop_interactive/0]).
%% Test suite API
-export([require/1, require/2,
get_config/1, get_config/2, get_config/3,
reload_config/1,
log/1, log/2, log/3, log/4,
print/1, print/2, print/3, print/4,
pal/1, pal/2, pal/3, pal/4,
capture_start/0, capture_stop/0, capture_get/0, capture_get/1,
fail/1, fail/2, comment/1, comment/2, make_priv_dir/0,
testcases/2, userdata/2, userdata/3,
timetrap/1, get_timetrap_info/0, sleep/1]).
%% New API for manipulating with config handlers
-export([add_config/2, remove_config/2]).
%% Other interface functions
-export([get_status/0, abort_current_testcase/1,
encrypt_config_file/2, encrypt_config_file/3,
decrypt_config_file/2, decrypt_config_file/3]).
-export([get_target_name/1]).
-export([parse_table/1, listenv/1]).
%%%-----------------------------------------------------------------
%%% @spec install(Opts) -> ok | {error,Reason}
%%% Opts = [Opt]
%%% Opt = {config,ConfigFiles} | {event_handler,Modules} |
%%% {decrypt,KeyOrFile}
%%% ConfigFiles = [ConfigFile]
%%% ConfigFile = string()
%%% Modules = [atom()]
%%% KeyOrFile = {key,Key} | {file,KeyFile}
%%% Key = string()
%%% KeyFile = string()
%%% @doc Install config files and event handlers.
%%%
%%% Run this function once before first test.
%%% %%%Example:
%%% install([{config,["config_node.ctc","config_user.ctc"]}]).
Note that this function is automatically run by the
%%% ct_run program.
Requires that ct:install/1 has been run first.
Suites (*_SUITE.erl) files must be stored in
%%% TestDir or TestDir/test. All suites
%%% will be compiled when test is run.
Opts.
%%% The options are the same as those used with the
%%% ct_runTestDir can be used to point out the path to
%%% a Suite. Note also that the option testcase
%%% corresponds to the -case option in the ct_run
%%% program. Configuration files specified in Opts will be
%%% installed automatically at startup.
run_test(Opts) ->
ct_run:run_test(Opts).
%%%-----------------------------------------------------------------
%%% @spec run_testspec(TestSpec) -> Result
%%% TestSpec = [term()]
%%% @doc Run test specified by TestSpec. The terms are
%%% the same as those used in test specification files.
run_testspec(TestSpec) ->
ct_run:run_testspec(TestSpec).
%%%-----------------------------------------------------------------
%%% @spec step(TestDir,Suite,Case) -> Result
%%% Case = atom()
%%%
%%% @doc Step through a test case with the debugger.
%%% @see run/3
step(TestDir,Suite,Case) ->
ct_run:step(TestDir,Suite,Case).
%%%-----------------------------------------------------------------
%%% @spec step(TestDir,Suite,Case,Opts) -> Result
%%% Case = atom()
%%% Opts = [Opt] | []
%%% Opt = config | keep_inactive
%%%
%%% @doc Step through a test case with the debugger. If the
%%% config option has been given, breakpoints will
%%% be set also on the configuration functions in Suite.
%%% @see run/3
step(TestDir,Suite,Case,Opts) ->
ct_run:step(TestDir,Suite,Case,Opts).
%%%-----------------------------------------------------------------
%%% @spec start_interactive() -> ok
%%%
%%% @doc Start CT in interactive mode.
%%%
%%% From this mode all test case support functions can be executed
%%% directly from the erlang shell. The interactive mode can also be
%%% started from the OS command line with ct_run -shell
%%% [-config File...].
If any functions using "required config data" (e.g. telnet or
%%% ftp functions) are to be called from the erlang shell, config data
%%% must first be required with ct:require/2.
Example:
%%% > ct:require(unix_telnet, unix).
%%% ok
%%% > ct_telnet:open(unix_telnet).
%%% {ok,<0.105.0>}
%%% > ct_telnet:cmd(unix_telnet, "ls .").
%%% {ok,["ls","file1 ...",...]}
Example: require the variable myvar:
%%% ok = ct:require(myvar)
In this case the config file must at least contain:
%%%
%%% {myvar,Value}.
%%%
%%% Example: require the variable myvar with
%%% subvariable sub1:
%%% ok = ct:require({myvar,sub1})
In this case the config file must at least contain:
%%%
%%% {myvar,[{sub1,Value}]}.
%%%
%%% @see require/2
%%% @see get_config/1
%%% @see get_config/2
%%% @see get_config/3
require(Required) ->
ct_config:require(Required).
%%%-----------------------------------------------------------------
%%% @spec require(Name,Required) -> ok | {error,Reason}
%%% Name = atom()
%%% Required = Key | {Key,SubKeys}
%%% Key = atom()
%%% SubKeys = SubKey | [SubKey]
%%% SubKey = atom()
%%%
%%% @doc Check if the required configuration is available, and give it
%%% a name.
%%%
%%% If the requested data is available, the main entry will be
%%% associated with Name so that the value of the element
%%% can be read with get_config/1,2 provided
%%% Name instead of the Key.
Example: Require one node with a telnet connection and an
%%% ftp connection. Name the node a:
ok =
%%% ct:require(a,{node,[telnet,ftp]}).
All references
%%% to this node may then use the node name. E.g. you can fetch a
%%% file over ftp like this:
%%% ok = ct:ftp_get(a,RemoteFile,LocalFile).
For this to work, the config file must at least contain:
%%%
%%% {node,[{telnet,IpAddr},
%%% {ftp,IpAddr}]}.
%%%
%%% @see require/1
%%% @see get_config/1
%%% @see get_config/2
%%% @see get_config/3
require(Name,Required) ->
ct_config:require(Name,Required).
%%%-----------------------------------------------------------------
%%% @spec get_config(Required) -> Value
%%% @equiv get_config(Required,undefined,[])
get_config(Required) ->
ct_config:get_config(Required,undefined,[]).
%%%-----------------------------------------------------------------
%%% @spec get_config(Required,Default) -> Value
%%% @equiv get_config(Required,Default,[])
get_config(Required,Default) ->
ct_config:get_config(Required,Default,[]).
%%%-----------------------------------------------------------------
%%% @spec get_config(Required,Default,Opts) -> ValueOrElement
%%% Required = KeyOrName | {KeyOrName,SubKey}
%%% KeyOrName = atom()
%%% SubKey = atom()
%%% Default = term()
%%% Opts = [Opt] | []
%%% Opt = element | all
%%% ValueOrElement = term() | Default
%%%
%%% @doc Read config data values.
%%%
%%% This function returns the matching value(s) or config element(s),
%%% given a config variable key or its associated name
%%% (if one has been specified with require/2 or a
%%% require statement).
Example, given the following config file:
%%%
%%% {unix,[{telnet,IpAddr},
%%% {username,Username},
%%% {password,Password}]}.
%%% get_config(unix,Default) ->
%%% [{telnet,IpAddr},
%%% {username,Username},
%%% {password,Password}]
%%% get_config({unix,telnet},Default) -> IpAddr
%%% get_config({unix,ftp},Default) -> Default
%%% get_config(unknownkey,Default) -> Default
If a config variable key has been associated with a name (by
%%% means of require/2 or a require statement), the name
%%% may be used instead of the key to read the value:
require(myhost,unix) -> ok
%%% get_config(myhost,Default) ->
%%% [{telnet,IpAddr},
%%% {username,Username},
%%% {password,Password}]
If a config variable is defined in multiple files and you want to
%%% access all possible values, use the all option. The
%%% values will be returned in a list and the order of the elements
%%% corresponds to the order that the config files were specified at
%%% startup.
If you want config elements (key-value tuples) returned as result
%%% instead of values, use the element option.
%%% The returned elements will then be on the form {KeyOrName,Value},
%%% or (in case a subkey has been specified)
%%% {{KeyOrName,SubKey},Value}
This function performs updating of the configuration data from which the %%% given configuration variable was read, and returns the (possibly) new %%% value of this variable.
%%%Note that if some variables were present in the configuration but are not loaded %%% using this function, they will be removed from the configuration table together %%% with their aliases.
%%% reload_config(Required)-> ct_config:reload_config(Required). %%%----------------------------------------------------------------- %%% @spec log(Format) -> ok %%% @equiv log(default,?STD_IMPORTANCE,Format,[]) log(Format) -> log(default,?STD_IMPORTANCE,Format,[]). %%%----------------------------------------------------------------- %%% @spec log(X1,X2) -> ok %%% X1 = Category | Importance | Format %%% X2 = Format | Args %%% @equiv log(Category,Importance,Format,Args) log(X1,X2) -> {Category,Importance,Format,Args} = if is_atom(X1) -> {X1,?STD_IMPORTANCE,X2,[]}; is_integer(X1) -> {default,X1,X2,[]}; is_list(X1) -> {default,?STD_IMPORTANCE,X1,X2} end, log(Category,Importance,Format,Args). %%%----------------------------------------------------------------- %%% @spec log(X1,X2,X3) -> ok %%% X1 = Category | Importance %%% X2 = Importance | Format %%% X3 = Format | Args %%% @equiv log(Category,Importance,Format,Args) log(X1,X2,X3) -> {Category,Importance,Format,Args} = if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[]}; is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3}; is_integer(X1) -> {default,X1,X2,X3} end, log(Category,Importance,Format,Args). %%%----------------------------------------------------------------- %%% @spec log(Category,Importance,Format,Args) -> ok %%% Category = atom() %%% Importance = integer() %%% Format = string() %%% Args = list() %%% %%% @doc Printout from a test case to the log file. %%% %%%This function is meant for printing a string directly from a %%% test case to the test case log file.
%%% %%%Default Category is default,
%%% default Importance is ?STD_IMPORTANCE,
%%% and default value for Args is [].
Please see the User's Guide for details on Category
%%% and Importance.
This function is meant for printing a string from a test case %%% to the console.
%%% %%%Default Category is default,
%%% default Importance is ?STD_IMPORTANCE,
%%% and default value for Args is [].
Please see the User's Guide for details on Category
%%% and Importance.
This function is meant for printing a string from a test case, %%% both to the test case log file and to the console.
%%% %%%Default Category is default,
%%% default Importance is ?STD_IMPORTANCE,
%%% and default value for Args is [].
Please see the User's Guide for details on Category
%%% and Importance.
capture_start/0).
%%%
%%% @see capture_start/0
%%% @see capture_get/1
capture_stop() ->
test_server:capture_stop().
%%%-----------------------------------------------------------------
%%% @spec capture_get() -> ListOfStrings
%%% ListOfStrings = [string()]
%%%
%%% @equiv capture_get([default])
capture_get() ->
%% remove default log printouts (e.g. ct:log/2 printouts)
capture_get([default]).
%%%-----------------------------------------------------------------
%%% @spec capture_get(ExclCategories) -> ListOfStrings
%%% ExclCategories = [atom()]
%%% ListOfStrings = [string()]
%%%
%%% @doc Return and purge the list of text strings buffered
%%% during the latest session of capturing printouts to stdout.
%%% With ExclCategories it's possible to specify
%%% log categories that should be ignored in ListOfStrings.
%%% If ExclCategories = [], no filtering takes place.
%%%
%%% @see capture_start/0
%%% @see capture_stop/0
%%% @see log/3
capture_get([ExclCat | ExclCategories]) ->
Strs = test_server:capture_get(),
CatsStr = [atom_to_list(ExclCat) |
[[$| | atom_to_list(EC)] || EC <- ExclCategories]],
{ok,MP} = re:compile("Reason.
fail(Reason) ->
try
exit({test_case_failed,Reason})
catch
Class:R ->
case erlang:get_stacktrace() of
[{?MODULE,fail,1,_}|Stk] -> ok;
Stk -> ok
end,
erlang:raise(Class, R, Stk)
end.
%%%-----------------------------------------------------------------
%%% @spec fail(Format, Args) -> void()
%%% Format = string()
%%% Args = list()
%%%
%%% @doc Terminate a test case with an error message specified
%%% by a format string and a list of values (used as arguments to
%%% io_lib:format/2).
fail(Format, Args) ->
try io_lib:format(Format, Args) of
Str ->
try
exit({test_case_failed,lists:flatten(Str)})
catch
Class:R ->
case erlang:get_stacktrace() of
[{?MODULE,fail,2,_}|Stk] -> ok;
Stk -> ok
end,
erlang:raise(Class, R, Stk)
end
catch
_:BadArgs ->
exit({BadArgs,{?MODULE,fail,[Format,Args]}})
end.
%%%-----------------------------------------------------------------
%%% @spec comment(Comment) -> void()
%%% Comment = term()
%%%
%%% @doc Print the given Comment in the comment field in
%%% the table on the test suite result page.
%%%
%%% If called several times, only the last comment is printed.
%%% The test case return value {comment,Comment}
%%% overwrites the string set by this function.
The Format and Args arguments are
%%% used in call to io_lib:format/2 in order to create
%%% the comment string. The behaviour of comment/2 is
%%% otherwise the same as the comment/1 function (see
%%% above for details).
The printout to parse would typically be the result of a
%%% select command in SQL. The returned
%%% Table is a list of tuples, where each tuple is a row
%%% in the table.
Heading is a tuple of strings representing the
%%% headings of each column in the table.
userdata
%%% in the list of tuples returned from Suite:suite/0.
userdata(TestDir, Suite) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
Info = (catch Suite:suite()),
get_userdata(Info, "suite/0")
end.
get_userdata({'EXIT',{Undef,_}}, Spec) when Undef == undef;
Undef == function_clause ->
{error,list_to_atom(Spec ++ " is not defined")};
get_userdata({'EXIT',Reason}, Spec) ->
{error,{list_to_atom("error in " ++ Spec),Reason}};
get_userdata(List, _) when is_list(List) ->
Fun = fun({userdata,Data}, Acc) -> [Data | Acc];
(_, Acc) -> Acc
end,
case lists:foldl(Fun, [], List) of
Terms ->
lists:flatten(lists:reverse(Terms))
end;
get_userdata(_BadTerm, Spec) ->
{error,list_to_atom(Spec ++ " must return a list")}.
%%%-----------------------------------------------------------------
%%% @spec userdata(TestDir, Suite, GroupOrCase) -> TCUserData | {error,Reason}
%%% TestDir = string()
%%% Suite = atom()
%%% GroupOrCase = {group,GroupName} | atom()
%%% GroupName = atom()
%%% TCUserData = [term()]
%%% Reason = term()
%%%
%%% @doc Returns any data specified with the tag userdata
%%% in the list of tuples returned from Suite:group(GroupName)
%%% or Suite:Case().
userdata(TestDir, Suite, {group,GroupName}) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
Info = (catch apply(Suite, group, [GroupName])),
get_userdata(Info, "group("++atom_to_list(GroupName)++")")
end;
userdata(TestDir, Suite, Case) when is_atom(Case) ->
case make_and_load(TestDir, Suite) of
E = {error,_} ->
E;
_ ->
Info = (catch apply(Suite, Case, [])),
get_userdata(Info, atom_to_list(Case)++"/0")
end.
%%%-----------------------------------------------------------------
%%% @spec get_status() -> TestStatus | {error,Reason} | no_tests_running
%%% TestStatus = [StatusElem]
%%% StatusElem = {current,{Suite,TestCase}} | {successful,Successful} |
%%% {failed,Failed} | {skipped,Skipped} | {total,Total}
%%% Suite = atom()
%%% TestCase = atom()
%%% Successful = integer()
%%% Failed = integer()
%%% Skipped = {UserSkipped,AutoSkipped}
%%% UserSkipped = integer()
%%% AutoSkipped = integer()
%%% Total = integer()
%%% Reason = term()
%%%
%%% @doc Returns status of ongoing test. The returned list contains info about
%%% which test case is currently executing, as well as counters for
%%% successful, failed, skipped, and total test cases so far.
get_status() ->
case get_testdata(curr_tc) of
{ok,TestCase} ->
case get_testdata(stats) of
{ok,{Ok,Failed,Skipped={UserSkipped,AutoSkipped}}} ->
[{current,TestCase},
{successful,Ok},
{failed,Failed},
{skipped,Skipped},
{total,Ok+Failed+UserSkipped+AutoSkipped}];
Err1 -> Err1
end;
Err2 -> Err2
end.
get_testdata(Key) ->
case catch ct_util:get_testdata(Key) of
{error,ct_util_server_not_running} ->
no_tests_running;
Error = {error,_Reason} ->
Error;
{'EXIT',_Reason} ->
no_tests_running;
Data ->
{ok,Data}
end.
%%%-----------------------------------------------------------------
%%% @spec abort_current_testcase(Reason) -> ok | {error,no_testcase_running}
%%% Reason = term()
%%%
%%% @doc When calling this function, the currently executing test case will be aborted. %%% It is the user's responsibility to know for sure which test case is currently %%% executing. The function is therefore only safe to call from a function which %%% has been called (or synchronously invoked) by the test case.
%%% %%%Reason, the reason for aborting the test case, is printed
%%% in the test case log.
This function encrypts the source config file with DES3 and
%%% saves the result in file EncryptFileName. The key,
%%% a string, must be available in a text file named
%%% .ct_config.crypt in the current directory, or the
%%% home directory of the user (it is searched for in that order).
See the Common Test User's Guide for information about using %%% encrypted config files when running tests.
%%%See the crypto application for details on DES3
%%% encryption/decryption.
This function encrypts the source config file with DES3 and
%%% saves the result in the target file EncryptFileName.
%%% The encryption key to use is either the value in
%%% {key,Key} or the value stored in the file specified
%%% by {file,File}.
See the Common Test User's Guide for information about using %%% encrypted config files when running tests.
%%%See the crypto application for details on DES3
%%% encryption/decryption.
This function decrypts EncryptFileName, previously
%%% generated with encrypt_config_file/2/3. The original
%%% file contents is saved in the target file. The encryption key, a
%%% string, must be available in a text file named
%%% .ct_config.crypt in the current directory, or the
%%% home directory of the user (it is searched for in that order).
This function decrypts EncryptFileName, previously
%%% generated with encrypt_config_file/2/3. The original
%%% file contents is saved in the target file. The key must have the
%%% the same value as that used for encryption.
This function loads configuration variables using the
%%% given callback module and configuration string. Callback module
%%% should be either loaded or present in the code part. Loaded
%%% configuration variables can later be removed using
%%% remove_config/2 function.
This function removes configuration variables (together with %%% their aliases) which were loaded with specified callback module and %%% configuration string.
remove_config(Callback, Config) -> ct_config:remove_config(Callback, Config). %%%----------------------------------------------------------------- %%% @spec timetrap(Time) -> ok %%% Time = {hours,Hours} | {minutes,Mins} | {seconds,Secs} | Millisecs | infinity | Func %%% Hours = integer() %%% Mins = integer() %%% Secs = integer() %%% Millisecs = integer() | float() %%% Func = {M,F,A} | fun() %%% M = atom() %%% F = atom() %%% A = list() %%% %%% @docUse this function to set a new timetrap for the running test case.
%%% If the argument is Func, the timetrap will be triggered
%%% when this function returns. Func may also return a new
%%% Time value, which in that case will be the value for the
%%% new timetrap.
Read info about the timetrap set for the current test case.
%%%
This function, similar to