aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/ct.erl
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/src/ct.erl')
-rw-r--r--lib/common_test/src/ct.erl332
1 files changed, 243 insertions, 89 deletions
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 6373634812..e369c9f4ef 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -51,6 +51,8 @@
-module(ct).
+-include("ct.hrl").
+
%% Command line user interface for running tests
-export([install/1, run/1, run/2, run/3,
run_test/1, run_testspec/1, step/3, step/4,
@@ -60,14 +62,15 @@
-export([require/1, require/2,
get_config/1, get_config/2, get_config/3,
reload_config/1,
- log/1, log/2, log/3,
- print/1, print/2, print/3,
- pal/1, pal/2, pal/3,
+ log/1, log/2, log/3, log/4,
+ print/1, print/2, print/3, print/4,
+ pal/1, pal/2, pal/3, pal/4,
capture_start/0, capture_stop/0, capture_get/0, capture_get/1,
fail/1, fail/2, comment/1, comment/2, make_priv_dir/0,
testcases/2, userdata/2, userdata/3,
timetrap/1, get_timetrap_info/0, sleep/1,
- notify/2, sync_notify/2]).
+ notify/2, sync_notify/2,
+ break/1, break/2, continue/0, continue/1]).
%% New API for manipulating with config handlers
-export([add_config/2, remove_config/2]).
@@ -151,8 +154,10 @@ run(TestDirs) ->
%%% {multiply_timetraps,M} | {scale_timetraps,Bool} |
%%% {repeat,N} | {duration,DurTime} | {until,StopTime} |
%%% {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
-%%% {refresh_logs,LogDir} | {logopts,LogOpts} | {basic_html,Bool} |
-%%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool}
+%%% {refresh_logs,LogDir} | {logopts,LogOpts} |
+%%% {verbosity,VLevels} | {basic_html,Bool} |
+%%% {ct_hooks, CTHs} | {enable_builtin_hooks,Bool} |
+%%% {noinput,Bool}
%%% TestDirs = [string()] | string()
%%% Suites = [string()] | [atom()] | string() | atom()
%%% Cases = [atom()] | atom()
@@ -183,6 +188,9 @@ run(TestDirs) ->
%%% DecryptFile = string()
%%% LogOpts = [LogOpt]
%%% LogOpt = no_nl | no_src
+%%% VLevels = VLevel | [{Category,VLevel}]
+%%% VLevel = integer()
+%%% Category = atom()
%%% CTHs = [CTHModule | {CTHModule, CTHInitArgs}]
%%% CTHModule = atom()
%%% CTHInitArgs = term()
@@ -266,27 +274,34 @@ stop_interactive() ->
%%%-----------------------------------------------------------------
%%% @spec require(Required) -> ok | {error,Reason}
-%%% Required = Key | {Key,SubKeys}
+%%% Required = Key | {Key,SubKeys} | {Key,SubKey,SubKeys}
%%% Key = atom()
%%% SubKeys = SubKey | [SubKey]
%%% SubKey = atom()
%%%
-%%% @doc Check if the required configuration is available.
+%%% @doc Check if the required configuration is available. It is possible
+%%% to specify arbitrarily deep tuples as <c>Required</c>. Note that it is
+%%% only the last element of the tuple which can be a list of <c>SubKey</c>s.
%%%
-%%% <p>Example: require the variable <code>myvar</code>:<br/>
-%%% <code>ok = ct:require(myvar)</code></p>
+%%% <p>Example 1: require the variable <code>myvar</code>:</p>
+%%% <pre>ok = ct:require(myvar).</pre>
%%%
%%% <p>In this case the config file must at least contain:</p>
-%%% <pre>
-%%% {myvar,Value}.</pre>
+%%% <pre>{myvar,Value}.</pre>
%%%
-%%% <p>Example: require the variable <code>myvar</code> with
-%%% subvariable <code>sub1</code>:<br/>
-%%% <code>ok = ct:require({myvar,sub1})</code></p>
+%%% <p>Example 2: require the key <code>myvar</code> with
+%%% subkeys <code>sub1</code> and <code>sub2</code>:</p>
+%%% <pre>ok = ct:require({myvar,[sub1,sub2]}).</pre>
%%%
%%% <p>In this case the config file must at least contain:</p>
-%%% <pre>
-%%% {myvar,[{sub1,Value}]}.</pre>
+%%% <pre>{myvar,[{sub1,Value},{sub2,Value}]}.</pre>
+%%%
+%%% <p>Example 3: require the key <code>myvar</code> with
+%%% subkey <code>sub1</code> with <code>subsub1</code>:</p>
+%%% <pre>ok = ct:require({myvar,sub1,sub2}).</pre>
+%%%
+%%% <p>In this case the config file must at least contain:</p>
+%%% <pre>{myvar,[{sub1,[{sub2,Value}]}]}.</pre>
%%%
%%% @see require/2
%%% @see get_config/1
@@ -298,30 +313,36 @@ require(Required) ->
%%%-----------------------------------------------------------------
%%% @spec require(Name,Required) -> ok | {error,Reason}
%%% Name = atom()
-%%% Required = Key | {Key,SubKeys}
+%%% Required = Key | {Key,SubKey} | {Key,SubKey,SubKey}
+%%% SubKey = Key
%%% Key = atom()
-%%% SubKeys = SubKey | [SubKey]
-%%% SubKey = atom()
%%%
%%% @doc Check if the required configuration is available, and give it
-%%% a name.
+%%% a name. The semantics for <c>Required</c> is the same as in
+%%% <c>required/1</c> except that it is not possible to specify a list
+%%% of <c>SubKey</c>s.
%%%
-%%% <p>If the requested data is available, the main entry will be
+%%% <p>If the requested data is available, the sub entry will be
%%% associated with <code>Name</code> so that the value of the element
%%% can be read with <code>get_config/1,2</code> provided
-%%% <code>Name</code> instead of the <code>Key</code>.</p>
+%%% <code>Name</code> instead of the whole <code>Required</code> term.</p>
%%%
%%% <p>Example: Require one node with a telnet connection and an
-%%% ftp connection. Name the node <code>a</code>:<br/> <code>ok =
-%%% ct:require(a,{node,[telnet,ftp]}).</code><br/> All references
-%%% to this node may then use the node name. E.g. you can fetch a
-%%% file over ftp like this:<br/>
-%%% <code>ok = ct:ftp_get(a,RemoteFile,LocalFile).</code></p>
+%%% ftp connection. Name the node <code>a</code>:
+%%% <pre>ok = ct:require(a,{machine,node}).</pre>
+%%% All references to this node may then use the node name.
+%%% E.g. you can fetch a file over ftp like this:</p>
+%%% <pre>ok = ct:ftp_get(a,RemoteFile,LocalFile).</pre>
%%%
%%% <p>For this to work, the config file must at least contain:</p>
-%%% <pre>
-%%% {node,[{telnet,IpAddr},
-%%% {ftp,IpAddr}]}.</pre>
+%%% <pre>{machine,[{node,[{telnet,IpAddr},{ftp,IpAddr}]}]}.</pre>
+%%%
+%%% <note>The behaviour of this function changed radically in common_test
+%%% 1.6.2. In order too keep some backwards compatability it is still possible
+%%% to do: <br/><c>ct:require(a,{node,[telnet,ftp]}).</c><br/>
+%%% This will associate the name <c>a</c> with the top level <c>node</c> entry.
+%%% For this to work, the config file must at least contain:<br/>
+%%% <c>{node,[{telnet,IpAddr},{ftp,IpAddr}]}.</c></note>
%%%
%%% @see require/1
%%% @see get_config/1
@@ -344,7 +365,7 @@ get_config(Required,Default) ->
%%%-----------------------------------------------------------------
%%% @spec get_config(Required,Default,Opts) -> ValueOrElement
-%%% Required = KeyOrName | {KeyOrName,SubKey}
+%%% Required = KeyOrName | {KeyOrName,SubKey} | {KeyOrName,SubKey,SubKey}
%%% KeyOrName = atom()
%%% SubKey = atom()
%%% Default = term()
@@ -362,25 +383,25 @@ get_config(Required,Default) ->
%%% <p>Example, given the following config file:</p>
%%% <pre>
%%% {unix,[{telnet,IpAddr},
-%%% {username,Username},
-%%% {password,Password}]}.</pre>
-%%% <p><code>get_config(unix,Default) ->
+%%% {user,[{username,Username},
+%%% {password,Password}]}]}.</pre>
+%%% <p><code>ct:get_config(unix,Default) ->
%%% [{telnet,IpAddr},
-%%% {username,Username},
-%%% {password,Password}]</code><br/>
-%%% <code>get_config({unix,telnet},Default) -> IpAddr</code><br/>
-%%% <code>get_config({unix,ftp},Default) -> Default</code><br/>
-%%% <code>get_config(unknownkey,Default) -> Default</code></p>
+%%% {user, [{username,Username},
+%%% {password,Password}]}]</code><br/>
+%%% <code>ct:get_config({unix,telnet},Default) -> IpAddr</code><br/>
+%%% <code>ct:get_config({unix,user,username},Default) -> Username</code><br/>
+%%% <code>ct:get_config({unix,ftp},Default) -> Default</code><br/>
+%%% <code>ct:get_config(unknownkey,Default) -> Default</code></p>
%%%
%%% <p>If a config variable key has been associated with a name (by
%%% means of <code>require/2</code> or a require statement), the name
%%% may be used instead of the key to read the value:</p>
%%%
-%%% <p><code>require(myhost,unix) -> ok</code><br/>
-%%% <code>get_config(myhost,Default) ->
-%%% [{telnet,IpAddr},
-%%% {username,Username},
-%%% {password,Password}]</code></p>
+%%% <p><code>ct:require(myuser,{unix,user}) -> ok.</code><br/>
+%%% <code>ct:get_config(myuser,Default) ->
+%%% [{username,Username},
+%%% {password,Password}]</code></p>
%%%
%%% <p>If a config variable is defined in multiple files and you want to
%%% access all possible values, use the <code>all</code> option. The
@@ -390,9 +411,7 @@ get_config(Required,Default) ->
%%%
%%% <p>If you want config elements (key-value tuples) returned as result
%%% instead of values, use the <code>element</code> option.
-%%% The returned elements will then be on the form <code>{KeyOrName,Value}</code>,
-%%% or (in case a subkey has been specified)
-%%% <code>{{KeyOrName,SubKey},Value}</code></p>
+%%% The returned elements will then be on the form <code>{Required,Value}</code></p>
%%%
%%% @see get_config/1
%%% @see get_config/2
@@ -403,7 +422,7 @@ get_config(Required,Default,Opts) ->
%%%-----------------------------------------------------------------
%%% @spec reload_config(Required) -> ValueOrElement
-%%% Required = KeyOrName | {KeyOrName,SubKey}
+%%% Required = KeyOrName | {KeyOrName,SubKey} | {KeyOrName,SubKey,SubKey}
%%% KeyOrName = atom()
%%% SubKey = atom()
%%% ValueOrElement = term()
@@ -422,25 +441,41 @@ reload_config(Required)->
%%%-----------------------------------------------------------------
%%% @spec log(Format) -> ok
-%%% @equiv log(default,Format,[])
+%%% @equiv log(default,50,Format,[])
log(Format) ->
- log(default,Format,[]).
+ log(default,?STD_IMPORTANCE,Format,[]).
%%%-----------------------------------------------------------------
%%% @spec log(X1,X2) -> ok
-%%% X1 = Category | Format
+%%% X1 = Category | Importance | Format
%%% X2 = Format | Args
-%%% @equiv log(Category,Format,Args)
+%%% @equiv log(Category,Importance,Format,Args)
log(X1,X2) ->
- {Category,Format,Args} =
- if is_atom(X1) -> {X1,X2,[]};
- is_list(X1) -> {default,X1,X2}
+ {Category,Importance,Format,Args} =
+ if is_atom(X1) -> {X1,?STD_IMPORTANCE,X2,[]};
+ is_integer(X1) -> {default,X1,X2,[]};
+ is_list(X1) -> {default,?STD_IMPORTANCE,X1,X2}
end,
- log(Category,Format,Args).
+ log(Category,Importance,Format,Args).
%%%-----------------------------------------------------------------
-%%% @spec log(Category,Format,Args) -> ok
+%%% @spec log(X1,X2,X3) -> ok
+%%% X1 = Category | Importance
+%%% X2 = Importance | Format
+%%% X3 = Format | Args
+%%% @equiv log(Category,Importance,Format,Args)
+log(X1,X2,X3) ->
+ {Category,Importance,Format,Args} =
+ if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[]};
+ is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3};
+ is_integer(X1) -> {default,X1,X2,X3}
+ end,
+ log(Category,Importance,Format,Args).
+
+%%%-----------------------------------------------------------------
+%%% @spec log(Category,Importance,Format,Args) -> ok
%%% Category = atom()
+%%% Importance = integer()
%%% Format = string()
%%% Args = list()
%%%
@@ -449,30 +484,52 @@ log(X1,X2) ->
%%% <p>This function is meant for printing a string directly from a
%%% test case to the test case log file.</p>
%%%
-%%% <p>Default <code>Category</code> is <code>default</code> and
-%%% default <code>Args</code> is <code>[]</code>.</p>
-log(Category,Format,Args) ->
- ct_logs:tc_log(Category,Format,Args).
+%%% <p>Default <code>Category</code> is <code>default</code>,
+%%% default <code>Importance</code> is <code>?STD_IMPORTANCE</code>,
+%%% and default value for <code>Args</code> is <code>[]</code>.</p>
+%%% <p>Please see the User's Guide for details on <code>Category</code>
+%%% and <code>Importance</code>.</p>
+log(Category,Importance,Format,Args) ->
+ ct_logs:tc_log(Category,Importance,Format,Args).
%%%-----------------------------------------------------------------
%%% @spec print(Format) -> ok
-%%% @equiv print(default,Format,[])
+%%% @equiv print(default,50,Format,[])
print(Format) ->
- print(default,Format,[]).
+ print(default,?STD_IMPORTANCE,Format,[]).
%%%-----------------------------------------------------------------
-%%% @equiv print(Category,Format,Args)
+%%% @spec print(X1,X2) -> ok
+%%% X1 = Category | Importance | Format
+%%% X2 = Format | Args
+%%% @equiv print(Category,Importance,Format,Args)
print(X1,X2) ->
- {Category,Format,Args} =
- if is_atom(X1) -> {X1,X2,[]};
- is_list(X1) -> {default,X1,X2}
+ {Category,Importance,Format,Args} =
+ if is_atom(X1) -> {X1,?STD_IMPORTANCE,X2,[]};
+ is_integer(X1) -> {default,X1,X2,[]};
+ is_list(X1) -> {default,?STD_IMPORTANCE,X1,X2}
end,
- print(Category,Format,Args).
+ print(Category,Importance,Format,Args).
%%%-----------------------------------------------------------------
-%%% @spec print(Category,Format,Args) -> ok
+%%% @spec print(X1,X2,X3) -> ok
+%%% X1 = Category | Importance
+%%% X2 = Importance | Format
+%%% X3 = Format | Args
+%%% @equiv print(Category,Importance,Format,Args)
+print(X1,X2,X3) ->
+ {Category,Importance,Format,Args} =
+ if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[]};
+ is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3};
+ is_integer(X1) -> {default,X1,X2,X3}
+ end,
+ print(Category,Importance,Format,Args).
+
+%%%-----------------------------------------------------------------
+%%% @spec print(Category,Importance,Format,Args) -> ok
%%% Category = atom()
+%%% Importance = integer()
%%% Format = string()
%%% Args = list()
%%%
@@ -481,33 +538,52 @@ print(X1,X2) ->
%%% <p>This function is meant for printing a string from a test case
%%% to the console.</p>
%%%
-%%% <p>Default <code>Category</code> is <code>default</code> and
-%%% default <code>Args</code> is <code>[]</code>.</p>
-print(Category,Format,Args) ->
- ct_logs:tc_print(Category,Format,Args).
+%%% <p>Default <code>Category</code> is <code>default</code>,
+%%% default <code>Importance</code> is <code>?STD_IMPORTANCE</code>,
+%%% and default value for <code>Args</code> is <code>[]</code>.</p>
+%%% <p>Please see the User's Guide for details on <code>Category</code>
+%%% and <code>Importance</code>.</p>
+print(Category,Importance,Format,Args) ->
+ ct_logs:tc_print(Category,Importance,Format,Args).
%%%-----------------------------------------------------------------
%%% @spec pal(Format) -> ok
-%%% @equiv pal(default,Format,[])
+%%% @equiv pal(default,50,Format,[])
pal(Format) ->
- pal(default,Format,[]).
+ pal(default,?STD_IMPORTANCE,Format,[]).
%%%-----------------------------------------------------------------
%%% @spec pal(X1,X2) -> ok
-%%% X1 = Category | Format
+%%% X1 = Category | Importance | Format
%%% X2 = Format | Args
-%%% @equiv pal(Category,Format,Args)
+%%% @equiv pal(Category,Importance,Format,Args)
pal(X1,X2) ->
- {Category,Format,Args} =
- if is_atom(X1) -> {X1,X2,[]};
- is_list(X1) -> {default,X1,X2}
+ {Category,Importance,Format,Args} =
+ if is_atom(X1) -> {X1,?STD_IMPORTANCE,X2,[]};
+ is_integer(X1) -> {default,X1,X2,[]};
+ is_list(X1) -> {default,?STD_IMPORTANCE,X1,X2}
end,
- pal(Category,Format,Args).
+ pal(Category,Importance,Format,Args).
%%%-----------------------------------------------------------------
-%%% @spec pal(Category,Format,Args) -> ok
+%%% @spec pal(X1,X2,X3) -> ok
+%%% X1 = Category | Importance
+%%% X2 = Importance | Format
+%%% X3 = Format | Args
+%%% @equiv pal(Category,Importance,Format,Args)
+pal(X1,X2,X3) ->
+ {Category,Importance,Format,Args} =
+ if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[]};
+ is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3};
+ is_integer(X1) -> {default,X1,X2,X3}
+ end,
+ pal(Category,Importance,Format,Args).
+
+%%%-----------------------------------------------------------------
+%%% @spec pal(Category,Importance,Format,Args) -> ok
%%% Category = atom()
+%%% Importance = integer()
%%% Format = string()
%%% Args = list()
%%%
@@ -516,10 +592,13 @@ pal(X1,X2) ->
%%% <p>This function is meant for printing a string from a test case,
%%% both to the test case log file and to the console.</p>
%%%
-%%% <p>Default <code>Category</code> is <code>default</code> and
-%%% default <code>Args</code> is <code>[]</code>.</p>
-pal(Category,Format,Args) ->
- ct_logs:tc_pal(Category,Format,Args).
+%%% <p>Default <code>Category</code> is <code>default</code>,
+%%% default <code>Importance</code> is <code>?STD_IMPORTANCE</code>,
+%%% and default value for <code>Args</code> is <code>[]</code>.</p>
+%%% <p>Please see the User's Guide for details on <code>Category</code>
+%%% and <code>Importance</code>.</p>
+pal(Category,Importance,Format,Args) ->
+ ct_logs:tc_pal(Category,Importance,Format,Args).
%%%-----------------------------------------------------------------
%%% @spec capture_start() -> ok
@@ -841,8 +920,9 @@ userdata(TestDir, Suite, Case) when is_atom(Case) ->
%%%-----------------------------------------------------------------
%%% @spec get_status() -> TestStatus | {error,Reason} | no_tests_running
%%% TestStatus = [StatusElem]
-%%% StatusElem = {current,{Suite,TestCase}} | {successful,Successful} |
+%%% StatusElem = {current,TestCaseInfo} | {successful,Successful} |
%%% {failed,Failed} | {skipped,Skipped} | {total,Total}
+%%% TestCaseInfo = {Suite,TestCase} | [{Suite,TestCase}]
%%% Suite = atom()
%%% TestCase = atom()
%%% Successful = integer()
@@ -854,7 +934,8 @@ userdata(TestDir, Suite, Case) when is_atom(Case) ->
%%% Reason = term()
%%%
%%% @doc Returns status of ongoing test. The returned list contains info about
-%%% which test case is currently executing, as well as counters for
+%%% which test case is currently executing (a list of cases when a
+%%% parallel test case group is executing), as well as counters for
%%% successful, failed, skipped, and total test cases so far.
get_status() ->
case get_testdata(curr_tc) of
@@ -879,6 +960,8 @@ get_testdata(Key) ->
Error;
{'EXIT',_Reason} ->
no_tests_running;
+ [CurrTC] when Key == curr_tc ->
+ {ok,CurrTC};
Data ->
{ok,Data}
end.
@@ -1072,3 +1155,74 @@ notify(Name,Data) ->
%%% @see //stdlib/gen_event
sync_notify(Name,Data) ->
ct_event:sync_notify(Name, Data).
+
+%%%-----------------------------------------------------------------
+%%% @spec break(Comment) -> ok | {error,Reason}
+%%% Comment = string()
+%%% Reason = {multiple_cases_running,TestCases}
+%%% TestCases = [atom()]
+%%%
+%%% @doc <p>This function will cancel all timetraps and pause the
+%%% execution of the current test case until the user calls the
+%%% <c>continue/0</c> function. It gives the user the opportunity
+%%% to interact with the erlang node running the tests, e.g. for
+%%% debugging purposes or for manually executing a part of the
+%%% test case. If a parallel group is executing, <c>break/2</c>
+%%% should be called instead.</p>
+break(Comment) ->
+ case get_testdata(curr_tc) of
+ {ok,{_,TestCase}} ->
+ test_server:break(?MODULE, Comment);
+ {ok,Cases} when is_list(Cases) ->
+ {error,{multiple_cases_running,
+ [TC || {_,TC} <- Cases]}};
+ Error ->
+ {error,Error}
+ end.
+
+%%%-----------------------------------------------------------------
+%%% @spec break(TestCase, Comment) -> ok | {error,Reason}
+%%% TestCase = atom()
+%%% Comment = string()
+%%% Reason = test_case_not_running
+%%%
+%%% @doc <p>This function works the same way as <c>break/1</c>,
+%%% only the <c>TestCase</c> argument makes it possible to
+%%% pause a test case executing in a parallel group. The
+%%% <c>continue/1</c> function should be used to resume
+%%% execution of <c>TestCase</c>.</p>
+break(TestCase, Comment) ->
+ case get_testdata(curr_tc) of
+ {ok,Cases} when is_list(Cases) ->
+ case lists:keymember(TestCase, 2, Cases) of
+ true ->
+ test_server:break(?MODULE, TestCase, Comment);
+ false ->
+ {error,test_case_not_running}
+ end;
+ {ok,{_,TestCase}} ->
+ test_server:break(?MODULE, TestCase, Comment);
+ Error ->
+ {error,Error}
+ end.
+
+%%%-----------------------------------------------------------------
+%%% @spec continue() -> ok
+%%%
+%%% @doc <p>This function must be called in order to continue after a
+%%% test case (not executing in a parallel group) has called
+%%% <c>break/1</c>.</p>
+continue() ->
+ test_server:continue().
+
+%%%-----------------------------------------------------------------
+%%% @spec continue(TestCase) -> ok
+%%% TestCase = atom()
+%%%
+%%% @doc <p>This function must be called in order to continue after a
+%%% test case has called <c>break/2</c>. If the paused test case,
+%%% <c>TestCase</c>, executes in a parallel group, this
+%%% function - rather than <c>continue/0</c> - must be used
+%%% in order to let the test case proceed.</p>
+continue(TestCase) ->
+ test_server:continue(TestCase).