From 02e8dc98b5327712f4ae00ac9fcf5feb1d856ba0 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 28 Aug 2012 17:38:15 +0200 Subject: Update the documentation for R15B02 OTP-10050 OTP-10069 OTP-10072 OTP-10087 OTP-9865 OTP-10049 OTP-10089 OTP-10145 OTP-9896 OTP-10135 OTP-10067 OTP-9625 OTP-10127 OTP-10172 OTP-10248 OTP-9625 OTP-10086 --- lib/common_test/doc/src/config_file_chapter.xml | 2 + lib/common_test/doc/src/ct_master_chapter.xml | 49 ++- lib/common_test/doc/src/ct_run.xml | 8 + lib/common_test/doc/src/run_test_chapter.xml | 530 ++++++++++++++++-------- lib/common_test/doc/src/write_test_chapter.xml | 109 ++++- lib/common_test/src/ct.erl | 238 ++++++----- lib/test_server/doc/src/test_server_ctrl.xml | 6 +- 7 files changed, 624 insertions(+), 318 deletions(-) (limited to 'lib') diff --git a/lib/common_test/doc/src/config_file_chapter.xml b/lib/common_test/doc/src/config_file_chapter.xml index e843ed3ba4..3e6fb21659 100644 --- a/lib/common_test/doc/src/config_file_chapter.xml +++ b/lib/common_test/doc/src/config_file_chapter.xml @@ -29,6 +29,8 @@ config_file_chapter.xml + +
General diff --git a/lib/common_test/doc/src/ct_master_chapter.xml b/lib/common_test/doc/src/ct_master_chapter.xml index f4f0ecad62..21deed099d 100644 --- a/lib/common_test/doc/src/ct_master_chapter.xml +++ b/lib/common_test/doc/src/ct_master_chapter.xml @@ -124,7 +124,8 @@

NodeRef = NodeAlias | node() | master

A NodeAlias (atom()) is used in a test specification as a - reference to a node name (so the actual node name only needs to be declared once). + reference to a node name (so the actual node name only needs to be declared once, + which can of course also be achieved using constants). The alias is declared with a node term:

{node, NodeAlias, NodeName}

@@ -141,30 +142,32 @@ CT Master:

-      {node, node1, ct_node@host_x}.
-      {node, node2, ct_node@host_y}.
- 
-      {logdir, master, "/home/test/master_logs"}.
-      {logdir, "/home/test/logs"}.
+      {define, 'Top', "/home/test"}.
+      {define, 'T1', "'Top'/t1"}.
+      {define, 'T2', "'Top'/t2"}.
+      {define, 'T3', "'Top'/t3"}.
+      {define, 'CfgFile', "config.cfg"}.
+      {define, 'Node', ct_node}.
+
+      {node, node1, 'Node@host_x'}.
+      {node, node2, 'Node@host_y'}.
+
+      {logdir, master, "'Top'/master_logs"}.
+      {logdir, "'Top'/logs"}.
       
-      {config, node1, "/home/test/t1/cfg/config.cfg"}.
-      {config, node2, "/home/test/t2/cfg/config.cfg"}.
-      {config, "/home/test/t3/cfg/config.cfg"}.
+      {config, node1, "'T1'/'CfgFile'"}.
+      {config, node2, "'T2'/'CfgFile'"}.
+      {config, "'T3'/'CfgFile'"}.
       
-      {alias, t1, "/home/test/t1"}.
-      {alias, t2, "/home/test/t2"}.
-      {alias, t3, "/home/test/t3"}.
+      {suites, node1, 'T1', all}.
+      {skip_suites, node1, 'T1', [t1B_SUITE,t1D_SUITE], "Not implemented"}.
+      {skip_cases, node1, 'T1', t1A_SUITE, [test3,test4], "Irrelevant"}.
+      {skip_cases, node1, 'T1', t1C_SUITE, [test1], "Ignore"}.
       
-      {suites, node1, t1, all}.
-      {skip_suites, node1, t1, [t1B_SUITE,t1D_SUITE], "Not implemented"}.
-      {skip_cases, node1, t1, t1A_SUITE, [test3,test4], "Irrelevant"}.
-      {skip_cases, node1, t1, t1C_SUITE, [test1], "Ignore"}.
+      {suites, node2, 'T2', [t2B_SUITE,t2C_SUITE]}.
+      {cases, node2, 'T2', t2A_SUITE, [test4,test1,test7]}.
       
-      {suites, node2, t2, [t2B_SUITE,t2C_SUITE]}.
-      {cases, node2, t2, t2A_SUITE, [test4,test1,test7]}.
-      
-      {skip_suites, t3, all, "Not implemented"}.
-
+ {skip_suites, 'T3', all, "Not implemented"}.

This example specifies the same tests as the original example. But now if started with a call to ct_master:run(TestSpecName), the @@ -190,10 +193,6 @@ name as the Common Test node in question (typically ct@somehost if started with the ct_run program), will be performed. Tests without explicit node association will always be performed too of course!

- -

It is recommended that absolute paths are used for log directories, - config files and test directory aliases in the test specifications so that - current working directory settings are not important.
diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 8061c840b0..9cc5495af7 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -36,6 +36,8 @@ OS command line. + +

The ct_run program is automatically installed with Erlang/OTP and Common Test (please see the Installation chapter in the Common @@ -72,6 +74,10 @@ following -erl_args on the command line. These directories are added to the code path normally (i.e. on specified form)

+

Exit status is set before the program ends. Value 0 indicates a successful + test result, 1 indicates one or more failed or auto-skipped test cases, and + 2 indicates test execution failure.

+

If ct_run is called with option:

-help

it prints all valid start flags to stdout.

@@ -112,6 +118,7 @@ [-basic_html] [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. CTHModuleN CTHOptsN] + [-exit_status ignore_config]
@@ -145,6 +152,7 @@ [-basic_html] [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. CTHModuleN CTHOptsN] + [-exit_status ignore_config]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 058b27d622..ea62df27cc 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -178,6 +178,8 @@ , switches off html enhancements that might not be compatible with older browsers. ]]>, makes it possible to modify aspects of the logging behaviour, see Log options below. + ]]>, sets verbosity levels + for printouts.

Directories passed to Common Test may have either relative or absolute paths.

 @@ -196,60 +198,73 @@ the current working directory of the Erlang Runtime System during the test run!

-

For more information about the ct_run program, see the - Installation chapter. -

-
- -
- Running tests from the Web based GUI - -

The web based GUI, VTS, is started with the ct_run - program. From the GUI you can load config files, and select - directories, suites and cases to run. You can also state the - config files, directories, suites and cases on the command line - when starting the web based GUI. -

- +

The ct_run program sets the exit status before shutting down. The following values + are defined:

- ct_run -vts - ]]> - -suite - -case ]]> + 0 indicates a successful testrun, i.e. one without failed or auto-skipped test cases. + 1 indicates that one or more test cases have failed, or have been auto-skipped. + 2 indicates that the test execution has failed because of e.g. compilation errors, an + illegal return value from an info function, etc. +

If auto-skipped test cases should not affect the exit status, you may change the default + behaviour using start flag:

+ 
-exit_status ignore_config
-

From the GUI you can run tests and view the result and the logs. +

For more information about the ct_run program, see the + Reference Manual and the + Installation chapter.

- -

Note that ct_run -vts will try to open the Common Test start - page in an existing web browser window or start the browser if it is - not running. Which browser should be started may be specified with - the browser start command option:

-

]]>

-

Example:

-

-

Note that the browser must run as a separate OS process or VTS will hang!

-

If no specific browser start command is specified, Firefox will - be the default browser on Unix platforms and Internet Explorer on Windows. - If Common Test fails to start a browser automatically, or 'none' is - specified as the value for -browser (i.e. -browser none), start your - favourite browser manually and type in the URL that Common Test - displays in the shell.

- +
Running tests from the Erlang shell or from an Erlang program

Common Test provides an Erlang API for running tests. The main (and most flexible) function for specifying and executing tests is called - ct:run_test/1. This function takes the same start parameters as - the ct_run program described above, only the flags are instead + ct:run_test/1. + This function takes the same start parameters as + the ct_run + program described above, only the flags are instead given as options in a list of key-value tuples. E.g. a test specified with ct_run like:

+

$ ct_run -suite ./my_SUITE -logdir ./results

is with ct:run_test/1 specified as:

1> ct:run_test([{suite,"./my_SUITE"},{logdir,"./results"}]).

-

For detailed documentation, please see the ct manual page.

+ +

The function returns the test result, represented by the tuple: + {Ok,Failed,{UserSkipped,AutoSkipped}}, where each element is an + integer. If test execution fails, the function returns the tuple: + {error,Reason}, where the term Reason explains the + failure.

+ +
+ Releasing the Erlang shell +

During execution of tests, started with + ct:run_test/1, + the Erlang shell process, controlling stdin, will remain the top + level process of the Common Test system of processes. The result + is that the Erlang shell is not available for interaction during + the test run. If this is not desirable, maybe because the shell is needed + for debugging purposes or for interaction with the SUT during test + execution, you may set the release_shell start option to + true (in the call to ct:run_test/1 or by + using the corresponding test specification term, see below). This will + make Common Test release the shell immediately after the test suite + compilation stage. To accomplish this, a test runner process + is spawned to take control of the test execution, and the effect is that + ct:run_test/1 returns the pid of this process rather than the + test result - which instead is printed to tty at the end of the test run.

+

Note that in order to use the + ct:break/1/2 and + ct:continue/0/1 functions, + release_shell must be set to true.

 +
+ +

For detailed documentation about + ct:run_test/1, + please see the + ct manual page.

@@ -353,31 +368,34 @@
- Using test specifications + Test Specifications

The most flexible way to specify what to test, is to use a so called test specification. A test specification is a sequence of - Erlang terms. The terms may be declared in a text file or passed - to the test server at runtime as a list - (see run_testspec/1 in the manual page - for ct). There are two general types of terms: - configuration terms and test specification terms.

+ Erlang terms. The terms are normally declared in a text file (see + ct:run_test/1), but + may also be passed to Common Test on the form of a list (see + ct:run_testspec/1). + There are two general types of terms: configuration terms and test + specification terms.

With configuration terms it is possible to e.g. label the test run (similar to ct_run -label), evaluate arbitrary expressions - before starting a test, import configuration - data (similar to - ct_run -config/-userconfig), specify HTML log directories (similar - to - ct_run -logdir), give aliases to test nodes and test - directories (to make a specification easier to read and - maintain), enable code coverage analysis (see - the Code Coverage - Analysis chapter) and specify event_handler plugins - (see the - Event Handling chapter). There is also a term for - specifying include directories that should be passed on to the - compiler when automatic compilation is performed (similar - to ct_run -include, see above).

+ before starting the test, import configuration data (similar to + ct_run -config/-userconfig), specify the top level HTML log + directory (similar to ct_run -logdir), enable code coverage + analysis (similar to ct_run -cover), install Common Test Hooks + (similar to ct_run -ch_hooks), install event_handler plugins + (similar to ct_run -event_handler), specify include directories + that should be passed to the compiler for automatic compilation + (similar to ct_run -include), disable the auto compilation + feature (similar to ct_run -no_auto_compile), set verbosity + levels (similar to ct_run -verbosity), and more.

+

Configuration terms can be combined with ct_run start flags, + or ct:run_test/1 options. The result will for some flags/options + and terms be that the values are merged (e.g. configuration files, + include directories, verbosity levels, silent connections), and for + others that the start flags/options override the test specification + terms (e.g. log directory, label, style sheet, auto compilation).

With test specification terms it is possible to state exactly which tests should run and in which order. A test term specifies either one or more suites, one or more test case groups, or one @@ -392,11 +410,12 @@ S, is a test of all cases in S. However, if a term specifying test case X and Y in S is merged with a term specifying case Z in S, the result is a test of X, Y and Z in S. To disable this - behaviour, it is possible in test specification to set the - merge_tests term to false.

+ behaviour, i.e. to instead perform each test sequentially in a "script-like" + manner, the term merge_tests can be set to false in + the test specification.

A test term can also specify one or more test suites, groups, or test cases to be skipped. Skipped suites, groups and cases - are not executed and show up in the HTML test log files as + are not executed and show up in the HTML log files as SKIPPED.

When a test case group is specified, the resulting test executes the @@ -429,12 +448,27 @@ Testing). The node parameters in the init term are only relevant in the latter (see the Large - Scale Testing chapter for information). For details on - the event_handler term, see the + Scale Testing chapter for information). For more information + about the various terms, please see the corresponding sections in the + User's Guide, such as e.g. the + ct_run + program for an overview of available start flags + (since most flags have a corresponding configuration term), and + more detailed explanation of e.g. + Logging + (for the verbosity, stylesheet and basic_html terms), + External Configuration Data + (for the config and userconfig terms), Event - Handling chapter.

+ Handling (for the event_handler term), + Common Test Hooks + (for the ct_hooks term), etc.

Config terms:

+      {merge_tests, Bool}.
+
+      {define, Constant, Value}.
+
       {node, NodeAlias, Node}.
 
       {init, InitOptions}.
@@ -443,6 +477,15 @@
       {label, Label}.
       {label, NodeRefs, Label}.
 
+      {verbosity, VerbosityLevels}.
+      {verbosity, NodeRefs, VerbosityLevels}.
+
+      {stylesheet, CSSFile}.
+      {stylesheet, NodeRefs, CSSFile}.
+
+      {silent_connections, ConnTypes}.
+      {silent_connections, NodeRefs, ConnTypes}.
+
       {multiply_timetraps, N}.
       {multiply_timetraps, NodeRefs, N}.
 
@@ -455,19 +498,23 @@
       {include, IncludeDirs}.
       {include, NodeRefs, IncludeDirs}.
 
+      {auto_compile, Bool},
+      {auto_compile, NodeRefs, Bool},
+
       {config, ConfigFiles}.
+      {config, ConfigDir, ConfigBaseNames}.
       {config, NodeRefs, ConfigFiles}.
+      {config, NodeRefs, ConfigDir, ConfigBaseNames}.
 
       {userconfig, {CallbackModule, ConfigStrings}}.
       {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}.
       
-      {alias, DirAlias, Dir}.
-
-      {merge_tests, Bool}.
-      
       {logdir, LogDir}.                                        
       {logdir, NodeRefs, LogDir}.
 
+      {logopts, LogOpts}.
+      {logopts, NodeRefs, LogOpts}.
+
       {create_priv_dir, PrivDirOption}.
       {create_priv_dir, NodeRefs, PrivDirOption}.
       
@@ -480,83 +527,176 @@
       {ct_hooks, NodeRefs, CTHModules}.
 
       {enable_builtin_hooks, Bool}.
-
+ + {basic_html, Bool}. + {basic_html, NodeRefs, Bool}. + + {release_shell, Bool}. +

Test terms:

-      {suites, DirRef, Suites}.                                
-      {suites, NodeRefs, DirRef, Suites}.
+      {suites, Dir, Suites}.                                
+      {suites, NodeRefs, Dir, Suites}.
       
-      {groups, DirRef, Suite, Groups}.
-      {groups, NodeRefsDirRef, Suite, Groups}.
+      {groups, Dir, Suite, Groups}.
+      {groups, NodeRefs, Dir, Suite, Groups}.
 
-      {groups, DirRef, Suite, GroupSpec, {cases,Cases}}.
-      {groups, NodeRefsDirRef, Suite, GroupSpec, {cases,Cases}}.
+      {groups, Dir, Suite, GroupSpec, {cases,Cases}}.
+      {groups, NodeRefs, Dir, Suite, GroupSpec, {cases,Cases}}.
 
-      {cases, DirRef, Suite, Cases}.                           
-      {cases, NodeRefs, DirRef, Suite, Cases}.
+      {cases, Dir, Suite, Cases}.                           
+      {cases, NodeRefs, Dir, Suite, Cases}.
 
-      {skip_suites, DirRef, Suites, Comment}.
-      {skip_suites, NodeRefs, DirRef, Suites, Comment}.
+      {skip_suites, Dir, Suites, Comment}.
+      {skip_suites, NodeRefs, Dir, Suites, Comment}.
 
-      {skip_groups, DirRef, Suite, GroupNames, Comment}.
-      {skip_groups, NodeRefs, DirRef, Suite, GroupNames, Comment}.
+      {skip_groups, Dir, Suite, GroupNames, Comment}.
+      {skip_groups, NodeRefs, Dir, Suite, GroupNames, Comment}.
       
-      {skip_cases, DirRef, Suite, Cases, Comment}.
-      {skip_cases, NodeRefs, DirRef, Suite, Cases, Comment}.
-
+ {skip_cases, Dir, Suite, Cases, Comment}. + {skip_cases, NodeRefs, Dir, Suite, Cases, Comment}. +

Types:

-      NodeAlias     = atom()
-      InitOptions   = term()
-      Node          = node()
-      NodeRef       = NodeAlias | Node | master
-      NodeRefs      = all_nodes | [NodeRef] | NodeRef
-      N             = integer()
-      Bool          = true | false
-      CoverSpecFile = string()
-      IncludeDirs   = string() | [string()]
-      ConfigFiles   = string() | [string()]
-      DirAlias      = atom()
-      Dir           = string()
-      LogDir        = string()
-      PrivDirOption = auto_per_run | auto_per_tc | manual_per_tc
-      EventHandlers = atom() | [atom()]
-      InitArgs      = [term()]
-      CTHModules    = [CTHModule | {CTHModule, CTHInitArgs} | {CTHModule, CTHInitArgs, CTHPriority}]
-      CTHModule     = atom()
-      CTHInitArgs   = term()
-      DirRef        = DirAlias | Dir
-      Suites        = atom() | [atom()] | all
-      Suite         = atom()
-      Groups        = GroupSpec | [GroupSpec] | all
-      GroupSpec     = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec}
-      GroupName     = atom()
-      GroupNames    = GroupName | [GroupName]
-      Cases         = atom() | [atom()] | all
-      Comment       = string() | ""
-
-

Example:

+ Bool = true | false + Constant = atom() + Value = term() + NodeAlias = atom() + Node = node() + NodeRef = NodeAlias | Node | master + NodeRefs = all_nodes | [NodeRef] | NodeRef + InitOptions = term() + Label = atom() | string() + VerbosityLevels = integer() | [{Category,integer()}] + Category = atom() + CSSFile = string() + ConnTypes = all | [atom()] + N = integer() + CoverSpecFile = string() + IncludeDirs = string() | [string()] + ConfigFiles = string() | [string()] + ConfigDir = string() + ConfigBaseNames = string() | [string()] + CallbackModule = atom() + ConfigStrings = string() | [string()] + LogDir = string() + LogOpts = [term()] + PrivDirOption = auto_per_run | auto_per_tc | manual_per_tc + EventHandlers = atom() | [atom()] + InitArgs = [term()] + CTHModules = [CTHModule | {CTHModule, CTHInitArgs} | {CTHModule, CTHInitArgs, CTHPriority}] + CTHModule = atom() + CTHInitArgs = term() + Dir = string() + Suites = atom() | [atom()] | all + Suite = atom() + Groups = GroupSpec | [GroupSpec] | all + GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec} + GroupName = atom() + GroupNames = GroupName | [GroupName] + Cases = atom() | [atom()] | all + Comment = string() | "" + +

The difference between the config terms above, is that with + ConfigDir, ConfigBaseNames is a list of base names, + i.e. without directory paths. ConfigFiles must be full names, + including paths. E.g, these two terms have the same meaning:

-      {logdir, "/home/test/logs"}.
+      {config, ["/home/testuser/tests/config/nodeA.cfg",
+                "/home/testuser/tests/config/nodeB.cfg"]}.
+
+      {config, "/home/testuser/tests/config", ["nodeA.cfg","nodeB.cfg"]}.
+ +

Any relative paths specified in the test specification, will be + relative to the directory which contains the test specification file, if + ct_run -spec TestSpecFile ... or + ct:run:test([{spec,TestSpecFile},...]) + executes the test. The path will be relative to the top level log directory, if + ct:run:testspec(TestSpec) executes the test.

 - {config, "/home/test/t1/cfg/config.cfg"}. - {config, "/home/test/t2/cfg/config.cfg"}. - {config, "/home/test/t3/cfg/config.cfg"}. +

The define term introduces a constant, which is used to + replace the name Constant with Value, wherever it's found in + the test specification. This replacement happens during an initial iteration + through the test specification. Constants may be used anywhere in the test + specification, e.g. in arbitrary lists and tuples, and even in strings + and inside the value part of other constant definitions! A constant can + also be part of a node name, but that is the only place where a constant + can be part of an atom.

+ +

For the sake of readability, the name of the constant must always + begin with an upper case letter, or a $, ?, or _. + This also means that it must always be single quoted (obviously, since + the constant name is actually an atom, not text).

 + +

The main benefit of constants is that they can be used to reduce the size + (and avoid repetition) of long strings, such as file paths. Compare these + terms:

+ + 
+      %% 1a. no constant
+      {config, "/home/testuser/tests/config", ["nodeA.cfg","nodeB.cfg"]}.
+      {suites, "/home/testuser/tests/suites", all}.
       
-      {alias, t1, "/home/test/t1"}.
-      {alias, t2, "/home/test/t2"}.
-      {alias, t3, "/home/test/t3"}.
+      %% 1b. with constant
+      {define, 'TESTDIR', "/home/testuser/tests"}.
+      {config, "'TESTDIR'/config", ["nodeA.cfg","nodeB.cfg"]}.
+      {suites, "'TESTDIR'/suites", all}.
+
+      %% 2a. no constants
+      {config, [testnode@host1, testnode@host2], "../config", ["nodeA.cfg","nodeB.cfg"]}.
+      {suites, [testnode@host1, testnode@host2], "../suites", [x_SUITE, y_SUITE]}.
+
+      %% 2b. with constants
+      {define, 'NODE', testnode}.
+      {define, 'NODES', ['NODE'@host1, 'NODE'@host2]}.
+      {config, 'NODES', "../config", ["nodeA.cfg","nodeB.cfg"]}.
+      {suites, 'NODES', "../suites", [x_SUITE, y_SUITE]}.
+ +

Constants make the test specification term alias, in previous + versions of Common Test, redundant. This term has been deprecated but will + remain supported in upcoming Common Test releases. Replacing alias + terms with define is strongly recommended though! Here's an example + of such a replacement:

+ + 
+      %% using the old alias term
+      {config, "/home/testuser/tests/config/nodeA.cfg"}.
+      {alias, suite_dir, "/home/testuser/tests/suites"}.
+      {groups, suite_dir, x_SUITE, group1}.
+
+      %% replacing with constants
+      {define, 'TestDir', "/home/testuser/tests"}.
+      {define, 'CfgDir', "'TestDir'/config"}.
+      {define, 'SuiteDir', "'TestDir'/suites"}.
+      {config, 'CfgDir', "nodeA.cfg"}.
+      {groups, 'SuiteDir', x_SUITE, group1}.
+ +

Actually, constants could well replace the node term too, but + this still has declarative value, mainly when used in combination + with NodeRefs == all_nodes (see types above).

+ +

Here follows a simple test specification example:

+ 
+      {define, 'Top', "/home/test"}.
+      {define, 'T1', "'Top'/t1"}.
+      {define, 'T2', "'Top'/t2"}.
+      {define, 'T3', "'Top'/t3"}.
+      {define, 'CfgFile', "config.cfg"}.
+
+      {logdir, "'Top'/logs"}.
       
-      {suites, t1, all}.
-      {skip_suites, t1, [t1B_SUITE,t1D_SUITE], "Not implemented"}.
-      {skip_cases, t1, t1A_SUITE, [test3,test4], "Irrelevant"}.
-      {skip_cases, t1, t1C_SUITE, [test1], "Ignore"}.
+      {config, ["'T1'/'CfgFile'", "'T2'/'CfgFile'", "'T3'/'CfgFile'"]}.
       
-      {suites, t2, [t2B_SUITE,t2C_SUITE]}.
-      {cases, t2, t2A_SUITE, [test4,test1,test7]}.
+      {suites, 'T1', all}.
+      {skip_suites, 'T1', [t1B_SUITE,t1D_SUITE], "Not implemented"}.
+      {skip_cases, 'T1', t1A_SUITE, [test3,test4], "Irrelevant"}.
+      {skip_cases, 'T1', t1C_SUITE, [test1], "Ignore"}.
       
-      {skip_suites, t3, all, "Not implemented"}.
-
+ {suites, 'T2', [t2B_SUITE,t2C_SUITE]}. + {cases, 'T2', t2A_SUITE, [test4,test1,test7]}. + + {skip_suites, 'T3', all, "Not implemented"}. +

The example specifies the following:

The specified logdir directory will be used for storing @@ -564,8 +704,6 @@ date and time). The variables in the specified test system config files will be imported for the test. - Aliases are given for three test system directories. The suites in - this example are stored in "/home/test/tX/test". The first test to run includes all suites for system t1. Excluded from the test are however the t1B and t1D suites. Also test cases test3 and test4 in t1A as well as the test1 case in t1C are excluded from @@ -590,8 +728,46 @@ If ct:run_test/1 is used for starting the tests, the relaxed scanner mode is enabled by means of the tuple: {allow_user_terms,true}

+ +
+ Running tests from the Web based GUI + +

The web based GUI, VTS, is started with the + ct_run + program. From the GUI you can load config files, and select + directories, suites and cases to run. You can also state the + config files, directories, suites and cases on the command line + when starting the web based GUI. +

+ + + ct_run -vts + ]]> + -suite + -case ]]> + + +

From the GUI you can run tests and view the result and the logs. +

+ +

Note that ct_run -vts will try to open the Common Test start + page in an existing web browser window or start the browser if it is + not running. Which browser should be started may be specified with + the browser start command option:

+

]]>

+

Example:

+

+

Note that the browser must run as a separate OS process or VTS will hang!

+

If no specific browser start command is specified, Firefox will + be the default browser on Unix platforms and Internet Explorer on Windows. + If Common Test fails to start a browser automatically, or 'none' is + specified as the value for -browser (i.e. -browser none), start your + favourite browser manually and type in the URL that Common Test + displays in the shell.

+
+ Log files

As the execution of the test suites proceed, events are logged in @@ -719,17 +895,30 @@

instead of each x printed on a new line, which is the default behaviour.

+
+ + Sorting HTML table columns +

By clicking the name in the column header of any table (e.g. "Ok", "Case", "Time", etc), + the table rows are sorted in whatever order makes sense for the type of value (e.g. + numerical for "Ok" or "Time", and alphabetical for "Case"). The sorting is performed + by means of JavaScript code, automatically inserted into the HTML log files. Common Test + uses the jQuery library and the + tablesorter plugin, with customized sorting + functions, for this implementation.

+
HTML Style Sheets -

Common Test uses a CSS file to control the look of the HTML - files generated during test runs. If, for some reason, the - log files are not displayed correctly in the HTML browser of your - choice, or you prefer the "pre Common Test v1.6 look" - of the log files (i.e. not using CSS), use the start flag/option - basic_html to revert to the old style.

+

Common Test uses an HTML Style Sheet (CSS file) to control the look of + the HTML log files generated during test runs. If, for some reason, the + log files are not displayed correctly in the browser of your + choice, or you prefer a more primitive ("pre Common Test v1.6") look + of the logs, use the start flag/option:

+ 
basic_html
+

This disables the use of Style Sheets, as well as JavaScripts (see + table sorting above).

Common Test includes an optional feature to allow user HTML style sheets for customizing printouts. The @@ -882,75 +1071,82 @@

Silent Connections -

The protocol handling processes in Common Test, implemented by ct_telnet, ct_ftp etc, - do verbose printing to the test case logs. This can be switched off by means - of the -silent_connections flag:

+

The protocol handling processes in Common Test, implemented by ct_telnet, + ct_ssh, ct_ftp etc, do verbose printing to the test case logs. This can be switched off + by means of the -silent_connections flag:

 	ct_run -silent_connections [conn_types]
-

where conn_types specifies telnet, ftp, rpc and/or snmp.

+

where conn_types specifies ssh, telnet, ftp, rpc and/or snmp.

Example:

-	ct_run ... -silent_connections telnet ftp
-

switches off logging for telnet and ftp connections.

+ ct_run ... -silent_connections ssh telnet +

switches off logging for ssh and telnet connections.

 	ct_run ... -silent_connections

switches off logging for all connection types.

-

Basic and important information such as opening and closing a connection, - fatal communication error and reconnection attempts will always be printed even - if logging has been suppressed for the connection type in question. However, operations - such as sending and receiving data may be performed silently.

+

Fatal communication error and reconnection attempts will always be printed even + if logging has been suppressed for the connection type in question. However, operations + such as sending and receiving data will be performed silently.

It is possible to also specify silent_connections in a test suite. This is accomplished by returning a tuple, {silent_connections,ConnTypes}, in the suite/0 or test case info list. If ConnTypes is a list of atoms - (telnet, ftp, rpc and/or snmp), output for any corresponding connections + (ssh, telnet, ftp, rpc and/or snmp), output for any corresponding connections will be suppressed. Full logging is per default enabled for any connection of type not specified in ConnTypes. Hence, if ConnTypes is the empty list, logging is enabled for all connections.

-

The silent_connections setting returned from a test case info function overrides, - for the test case in question, any setting made with suite/0 (which is the setting - used for all cases in the suite). Example:

+

Example:

 	
 	-module(my_SUITE).
+
+	suite() -> [..., {silent_connections,[telnet,ssh]}, ...].
+
 	...
-	suite() -> [..., {silent_connections,[telnet,ftp]}, ...].
-	...
+
 	my_testcase1() ->
-	[{silent_connections,[ftp]}].
+	    [{silent_connections,[ssh]}].
+
 	my_testcase1(_) ->
-	...
+	    ...
+
 	my_testcase2(_) ->
-	...
+	    ...

In this example, suite/0 tells Common Test to suppress - printouts from telnet and ftp connections. This is valid for + printouts from telnet and ssh connections. This is valid for all test cases. However, my_testcase1/0 specifies that - for this test case, only ftp should be silent. The result is + for this test case, only ssh should be silent. The result is that my_testcase1 will get telnet info (if any) printed - in the log, but not ftp info. my_testcase2 will get no + in the log, but not ssh info. my_testcase2 will get no info from either connection printed.

-

The -silent_connections tag (or - silent_connections tagged tuple in the call to - ct:run_test/1) overrides any settings in the test - suite.

+

silent_connections may also be specified with a term + in a test specification + (see Test + Specifications). Connections provided with the + silent_connections start flag/option, will be merged with + any connections listed in the test specification.

+ +

The silent_connections start flag/option and test + specification term, overrides any settings made by the info functions + inside the test suite.

-

Note that in the current Common Test version, the +

Note that in the current Common Test version, the silent_connections feature only works for telnet - connections. Support for other connection types will be added - in future Common Test versions.

+ and ssh connections! Support for other connection types will be added + in future Common Test versions.

diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 1fae50577e..90c08032ec 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -47,7 +47,7 @@ module for details about these functions.

The CT application also includes other modules named - ]]> that + ]]> that provide various support, mainly simplified use of communication protocols such as rpc, snmp, ftp, telnet, etc.

@@ -934,6 +934,99 @@ returned time.

+
+ + Logging - categories and verbosity levels +

Common Test provides three main functions for printing strings:

+ + ct:log(Category, Importance, Format, Args) + ct:print(Category, Importance, Format, Args) + ct:pal(Category, Importance, Format, Args) + +

The log/1/2/3/4 function will print a string to the test case + log file. The print/1/2/3/4 function will print the string to screen, + and the pal/1/2/3/4 function will print the same string both to file and + screen. (The functions are documented in the ct reference manual).

+ +

The optional Category argument may be used to categorize the + log printout, and categories can be used for two things:

+ + To compare the importance of the printout to a specific + verbosity level, and + to format the printout according to a user specific HTML + Style Sheet (CSS). + + +

The Importance argument specifies a level of importance + which, compared to a verbosity level (general and/or set per category), + determines if the printout should be visible or not. Importance + is an arbitrary integer in the range 0..99. Pre-defined constants + exist in the ct.hrl header file. The default importance level, + ?STD_IMPORTANCE (used if the Importance argument is not + provided), is 50. This is also the importance used for standard IO, e.g. + from printouts made with io:format/2, io:put_chars/1, etc.

+ +

Importance is compared to a verbosity level set by means of the + verbosity start flag/option. The verbosity level can be set per + category and/or generally. The default verbosity level, ?STD_VERBOSITY, + is 50, i.e. all standard IO gets printed. If a lower verbosity level is set, + standard IO printouts will be ignored. Common Test performs the following test:

+ 
Importance >= (100-VerbosityLevel)
+

This also means that verbosity level 0 effectively turns all logging off + (with the exception of printouts made by Common Test itself).

+ +

The general verbosity level is not associated with any particular + category. This level sets the threshold for the standard IO printouts, + uncategorized ct:log/print/pal printouts, as well as + printouts for categories with undefined verbosity level.

+ +

Example:

+ 
+
+      Some printouts during test case execution:
+
+        io:format("1. Standard IO, importance = ~w~n", [?STD_IMPORTANCE]),
+        ct:log("2. Uncategorized, importance = ~w", [?STD_IMPORTANCE]),
+        ct:log(info, "3. Categorized info, importance = ~w", [?STD_IMPORTANCE]]),
+        ct:log(info, ?LOW_IMPORTANCE, "4. Categorized info, importance = ~w", [?LOW_IMPORTANCE]),
+        ct:log(error, "5. Categorized error, importance = ~w", [?HI_IMPORTANCE]),
+        ct:log(error, ?HI_IMPORTANCE, "6. Categorized error, importance = ~w", [?MAX_IMPORTANCE]),
+
+      If starting the test without specifying any verbosity levels:
+
+        $ ct_run ...
+
+      the following gets printed:
+      
+        1. Standard IO, importance = 50
+        2. Uncategorized, importance = 50
+        3. Categorized info, importance = 50
+        5. Categorized error, importance = 75
+        6. Categorized error, importance = 99
+
+      If starting the test with:
+        
+        $ ct_run -verbosity 1 and info 75
+      
+      the following gets printed:
+
+        3. Categorized info, importance = 50
+	4. Categorized info, importance = 25
+        6. Categorized error, importance = 99
+
+ +

How categories can be mapped to CSS tags is documented in the + Running Tests + chapter.

+ +

The Format and Args arguments in ct:log/print/pal are + always passed on to the io:format/3 function in stdlib + (please see the io manual page for details).

+ +

For more information about log files, please see the + Running Tests chapter.

+
+
Illegal dependencies @@ -944,7 +1037,6 @@ Erlang/OTP test suites.

- Depending on current directory, and writing there:

This is a common error in test suites. It is assumed that @@ -956,19 +1048,10 @@

- Depending on the Clearcase (file version control system) - paths and files:

- -

The test suites are stored in Clearcase but are not - (necessarily) run within this environment. The directory - structure may vary from test run to test run. -

- - Depending on execution order:

-

During development of test suites, no assumption should be made - (preferrably) about the execution order of the test cases or suites. +

During development of test suites, no assumption should preferrably + be made about the execution order of the test cases or suites. E.g. a test case should not assume that a server it depends on, has already been started by a previous test case. There are several reasons for this: diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index aa8813c391..455fd1e4f1 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -25,24 +25,24 @@ %%% %%%

Test Suite Support Macros

%%% -%%%

The config macro is defined in ct.hrl. This +%%%

The config macro is defined in ct.hrl. This %%% macro should be used to retrieve information from the -%%% Config variable sent to all test cases. It is used with two +%%% Config variable sent to all test cases. It is used with two %%% arguments, where the first is the name of the configuration -%%% variable you wish to retrieve, and the second is the Config +%%% variable you wish to retrieve, and the second is the Config %%% variable supplied to the test case.

%%% %%%

Possible configuration variables include:

%%%
    -%%%
  • data_dir - Data file directory.
    • -%%%
  • priv_dir - Scratch file directory.
    • -%%%
  • Whatever added by init_per_suite/1 or -%%% init_per_testcase/2 in the test suite.
    • +%%%
  • data_dir - Data file directory.
    • +%%%
  • priv_dir - Scratch file directory.
    • +%%%
  • Whatever added by init_per_suite/1 or +%%% init_per_testcase/2 in the test suite.
    • %%%
%%% @type var_name() = atom(). A variable name which is specified when -%%% ct:require/2 is called, -%%% e.g. ct:require(mynodename,{node,[telnet]}) +%%% ct:require/2 is called, +%%% e.g. ct:require(mynodename,{node,[telnet]}) %%% %%% @type target_name() = var_name(). The name of a target. %%% @@ -99,10 +99,10 @@ %%%

Run this function once before first test.

%%% %%%

Example:
-%%% install([{config,["config_node.ctc","config_user.ctc"]}]).

+%%% install([{config,["config_node.ctc","config_user.ctc"]}]).

%%% %%%

Note that this function is automatically run by the -%%% ct_run program.

+%%% ct_run program.

install(Opts) -> ct_run:install(Opts). @@ -115,10 +115,10 @@ install(Opts) -> %%% %%% @doc Run the given test case(s). %%% -%%%

Requires that ct:install/1 has been run first.

+%%%

Requires that ct:install/1 has been run first.

%%% %%%

Suites (*_SUITE.erl) files must be stored in -%%% TestDir or TestDir/test. All suites +%%% TestDir or TestDir/test. All suites %%% will be compiled when test is run.

run(TestDir,Suite,Cases) -> ct_run:run(TestDir,Suite,Cases). @@ -201,25 +201,32 @@ run(TestDirs) -> %%% AutoSkipped = integer() %%% TestRunnerPid = pid() %%% Reason = term() -%%% @doc

Run tests as specified by the combination of options in Opts. +%%% @doc

Run tests as specified by the combination of options in Opts. %%% The options are the same as those used with the -%%% ct_run program. -%%% Note that here a TestDir can be used to point out the path to -%%% a Suite. Note also that the option testcase -%%% corresponds to the -case option in the ct_run -%%% program. Configuration files specified in Opts will be +%%% ct_run program. +%%% Note that here a TestDir can be used to point out the path to +%%% a Suite. Note also that the option testcase +%%% corresponds to the -case option in the ct_run +%%% program. Configuration files specified in Opts will be %%% installed automatically at startup.

-%%%

TestRunnerPid is returned if release_shell == true -%%% (see the User's Guide for details).

-%%%

Reason indicates what type of error has been encountered.

+%%%

TestRunnerPid is returned if release_shell == true +%%% (see break/1 for details).

+%%%

Reason indicates what type of error has been encountered.

run_test(Opts) -> ct_run:run_test(Opts). %%%----------------------------------------------------------------- %%% @spec run_testspec(TestSpec) -> Result %%% TestSpec = [term()] -%%% @doc Run test specified by TestSpec. The terms are +%%% Result = {Ok,Failed,{UserSkipped,AutoSkipped}} | {error,Reason} +%%% Ok = integer() +%%% Failed = integer() +%%% UserSkipped = integer() +%%% AutoSkipped = integer() +%%% Reason = term() +%%% @doc Run test specified by TestSpec. The terms are %%% the same as those used in test specification files. +%%%

Reason indicates what type of error has been encountered.

run_testspec(TestSpec) -> ct_run:run_testspec(TestSpec). @@ -239,8 +246,8 @@ step(TestDir,Suite,Case) -> %%% Opt = config | keep_inactive %%% %%% @doc Step through a test case with the debugger. If the -%%% config option has been given, breakpoints will -%%% be set also on the configuration functions in Suite. +%%% config option has been given, breakpoints will +%%% be set also on the configuration functions in Suite. %%% @see run/3 step(TestDir,Suite,Case,Opts) -> ct_run:step(TestDir,Suite,Case,Opts). @@ -252,20 +259,20 @@ step(TestDir,Suite,Case,Opts) -> %%% %%%

From this mode all test case support functions can be executed %%% directly from the erlang shell. The interactive mode can also be -%%% started from the OS command line with ct_run -shell -%%% [-config File...].

+%%% started from the OS command line with ct_run -shell +%%% [-config File...].

%%% %%%

If any functions using "required config data" (e.g. telnet or %%% ftp functions) are to be called from the erlang shell, config data -%%% must first be required with ct:require/2.

+%%% must first be required with ct:require/2.

%%% %%%

Example:
-%%% > ct:require(unix_telnet, unix).
-%%% ok
-%%% > ct_telnet:open(unix_telnet).
-%%% {ok,<0.105.0>}
-%%% > ct_telnet:cmd(unix_telnet, "ls .").
-%%% {ok,["ls","file1 ...",...]}

+%%% > ct:require(unix_telnet, unix).
+%%% ok
+%%% > ct_telnet:open(unix_telnet).
+%%% {ok,<0.105.0>}
+%%% > ct_telnet:cmd(unix_telnet, "ls .").
+%%% {ok,["ls","file1 ...",...]}

start_interactive() -> ct_util:start(interactive). @@ -292,21 +299,21 @@ stop_interactive() -> %%% to specify arbitrarily deep tuples as Required. Note that it is %%% only the last element of the tuple which can be a list of SubKeys. %%% -%%%

Example 1: require the variable myvar:

+%%%

Example 1: require the variable myvar:

%%% 
ok = ct:require(myvar).
%%% %%%

In this case the config file must at least contain:

%%% 
{myvar,Value}.
%%% -%%%

Example 2: require the key myvar with -%%% subkeys sub1 and sub2:

+%%%

Example 2: require the key myvar with +%%% subkeys sub1 and sub2:

%%% 
ok = ct:require({myvar,[sub1,sub2]}).
%%% %%%

In this case the config file must at least contain:

%%% 
{myvar,[{sub1,Value},{sub2,Value}]}.
%%% -%%%

Example 3: require the key myvar with -%%% subkey sub1 with subsub1:

+%%%

Example 3: require the key myvar with +%%% subkey sub1 with subsub1:

%%% 
ok = ct:require({myvar,sub1,sub2}).
%%% %%%

In this case the config file must at least contain:

@@ -332,12 +339,12 @@ require(Required) -> %%% of SubKeys. %%% %%%

If the requested data is available, the sub entry will be -%%% associated with Name so that the value of the element -%%% can be read with get_config/1,2 provided -%%% Name instead of the whole Required term.

+%%% associated with Name so that the value of the element +%%% can be read with get_config/1,2 provided +%%% Name instead of the whole Required term.

%%% %%%

Example: Require one node with a telnet connection and an -%%% ftp connection. Name the node a: +%%% ftp connection. Name the node a: %%% 

ok = ct:require(a,{machine,node}).
%%% All references to this node may then use the node name. %%% E.g. you can fetch a file over ftp like this:

@@ -386,7 +393,7 @@ get_config(Required,Default) -> %%% %%%

This function returns the matching value(s) or config element(s), %%% given a config variable key or its associated name -%%% (if one has been specified with require/2 or a +%%% (if one has been specified with require/2 or a %%% require statement).

%%% %%%

Example, given the following config file:

@@ -394,33 +401,33 @@ get_config(Required,Default) -> %%% {unix,[{telnet,IpAddr}, %%% {user,[{username,Username}, %%% {password,Password}]}]}. -%%%

ct:get_config(unix,Default) -> +%%%

ct:get_config(unix,Default) -> %%% [{telnet,IpAddr}, %%% {user, [{username,Username}, -%%% {password,Password}]}]
-%%% ct:get_config({unix,telnet},Default) -> IpAddr
-%%% ct:get_config({unix,user,username},Default) -> Username
-%%% ct:get_config({unix,ftp},Default) -> Default
-%%% ct:get_config(unknownkey,Default) -> Default

+%%% {password,Password}]}]
+%%% ct:get_config({unix,telnet},Default) -> IpAddr
+%%% ct:get_config({unix,user,username},Default) -> Username
+%%% ct:get_config({unix,ftp},Default) -> Default
+%%% ct:get_config(unknownkey,Default) -> Default

%%% %%%

If a config variable key has been associated with a name (by -%%% means of require/2 or a require statement), the name +%%% means of require/2 or a require statement), the name %%% may be used instead of the key to read the value:

%%% -%%%

ct:require(myuser,{unix,user}) -> ok.
-%%% ct:get_config(myuser,Default) -> +%%%

ct:require(myuser,{unix,user}) -> ok.
+%%% ct:get_config(myuser,Default) -> %%% [{username,Username}, -%%% {password,Password}]

+%%% {password,Password}]

%%% %%%

If a config variable is defined in multiple files and you want to -%%% access all possible values, use the all option. The +%%% access all possible values, use the all option. The %%% values will be returned in a list and the order of the elements %%% corresponds to the order that the config files were specified at %%% startup.

%%% %%%

If you want config elements (key-value tuples) returned as result -%%% instead of values, use the element option. -%%% The returned elements will then be on the form {Required,Value}

+%%% instead of values, use the element option. +%%% The returned elements will then be on the form {Required,Value}

%%% %%% @see get_config/1 %%% @see get_config/2 @@ -493,11 +500,11 @@ log(X1,X2,X3) -> %%%

This function is meant for printing a string directly from a %%% test case to the test case log file.

%%% -%%%

Default Category is default, -%%% default Importance is ?STD_IMPORTANCE, -%%% and default value for Args is [].

-%%%

Please see the User's Guide for details on Category -%%% and Importance.

+%%%

Default Category is default, +%%% default Importance is ?STD_IMPORTANCE, +%%% and default value for Args is [].

+%%%

Please see the User's Guide for details on Category +%%% and Importance.

log(Category,Importance,Format,Args) -> ct_logs:tc_log(Category,Importance,Format,Args). @@ -547,11 +554,11 @@ print(X1,X2,X3) -> %%%

This function is meant for printing a string from a test case %%% to the console.

%%% -%%%

Default Category is default, -%%% default Importance is ?STD_IMPORTANCE, -%%% and default value for Args is [].

-%%%

Please see the User's Guide for details on Category -%%% and Importance.

+%%%

Default Category is default, +%%% default Importance is ?STD_IMPORTANCE, +%%% and default value for Args is [].

+%%%

Please see the User's Guide for details on Category +%%% and Importance.

print(Category,Importance,Format,Args) -> ct_logs:tc_print(Category,Importance,Format,Args). @@ -601,11 +608,11 @@ pal(X1,X2,X3) -> %%%

This function is meant for printing a string from a test case, %%% both to the test case log file and to the console.

%%% -%%%

Default Category is default, -%%% default Importance is ?STD_IMPORTANCE, -%%% and default value for Args is [].

-%%%

Please see the User's Guide for details on Category -%%% and Importance.

+%%%

Default Category is default, +%%% default Importance is ?STD_IMPORTANCE, +%%% and default value for Args is [].

+%%%

Please see the User's Guide for details on Category +%%% and Importance.

pal(Category,Importance,Format,Args) -> ct_logs:tc_pal(Category,Importance,Format,Args). @@ -624,7 +631,7 @@ capture_start() -> %%% @spec capture_stop() -> ok %%% %%% @doc Stop capturing text strings (a session started with -%%% capture_start/0). +%%% capture_start/0). %%% %%% @see capture_start/0 %%% @see capture_get/1 @@ -647,9 +654,9 @@ capture_get() -> %%% %%% @doc Return and purge the list of text strings buffered %%% during the latest session of capturing printouts to stdout. -%%% With ExclCategories it's possible to specify -%%% log categories that should be ignored in ListOfStrings. -%%% If ExclCategories = [], no filtering takes place. +%%% With ExclCategories it's possible to specify +%%% log categories that should be ignored in ListOfStrings. +%%% If ExclCategories = [], no filtering takes place. %%% %%% @see capture_start/0 %%% @see capture_stop/0 @@ -674,7 +681,7 @@ capture_get([]) -> %%% Reason = term() %%% %%% @doc Terminate a test case with the given error -%%% Reason. +%%% Reason. fail(Reason) -> try exit({test_case_failed,Reason}) @@ -694,7 +701,7 @@ fail(Reason) -> %%% %%% @doc Terminate a test case with an error message specified %%% by a format string and a list of values (used as arguments to -%%% io_lib:format/2). +%%% io_lib:format/2). fail(Format, Args) -> try io_lib:format(Format, Args) of Str -> @@ -717,11 +724,11 @@ fail(Format, Args) -> %%% @spec comment(Comment) -> void() %%% Comment = term() %%% -%%% @doc Print the given Comment in the comment field in +%%% @doc Print the given Comment in the comment field in %%% the table on the test suite result page. %%% %%%

If called several times, only the last comment is printed. -%%% The test case return value {comment,Comment} +%%% The test case return value {comment,Comment} %%% overwrites the string set by this function.

comment(Comment) when is_list(Comment) -> Formatted = @@ -744,10 +751,10 @@ comment(Comment) -> %%% @doc Print the formatted string in the comment field in %%% the table on the test suite result page. %%% -%%%

The Format and Args arguments are -%%% used in call to io_lib:format/2 in order to create -%%% the comment string. The behaviour of comment/2 is -%%% otherwise the same as the comment/1 function (see +%%%

The Format and Args arguments are +%%% used in call to io_lib:format/2 in order to create +%%% the comment string. The behaviour of comment/2 is +%%% otherwise the same as the comment/1 function (see %%% above for details).

comment(Format, Args) when is_list(Format), is_list(Args) -> Formatted = @@ -792,11 +799,11 @@ get_target_name(Handle) -> %%% @doc Parse the printout from an SQL table and return a list of tuples. %%% %%%

The printout to parse would typically be the result of a -%%% select command in SQL. The returned -%%% Table is a list of tuples, where each tuple is a row +%%% select command in SQL. The returned +%%% Table is a list of tuples, where each tuple is a row %%% in the table.

%%% -%%%

Heading is a tuple of strings representing the +%%%

Heading is a tuple of strings representing the %%% headings of each column in the table.

parse_table(Data) -> ct_util:parse_table(Data). @@ -868,8 +875,8 @@ make_and_load(Dir, Suite) -> %%% SuiteUserData = [term()] %%% Reason = term() %%% -%%% @doc Returns any data specified with the tag userdata -%%% in the list of tuples returned from Suite:suite/0. +%%% @doc Returns any data specified with the tag userdata +%%% in the list of tuples returned from Suite:suite/0. userdata(TestDir, Suite) -> case make_and_load(TestDir, Suite) of E = {error,_} -> @@ -904,9 +911,9 @@ get_userdata(_BadTerm, Spec) -> %%% TCUserData = [term()] %%% Reason = term() %%% -%%% @doc Returns any data specified with the tag userdata -%%% in the list of tuples returned from Suite:group(GroupName) -%%% or Suite:Case(). +%%% @doc Returns any data specified with the tag userdata +%%% in the list of tuples returned from Suite:group(GroupName) +%%% or Suite:Case(). userdata(TestDir, Suite, {group,GroupName}) -> case make_and_load(TestDir, Suite) of E = {error,_} -> @@ -986,7 +993,7 @@ get_testdata(Key) -> %%% executing. The function is therefore only safe to call from a function which %%% has been called (or synchronously invoked) by the test case.

%%% -%%%

Reason, the reason for aborting the test case, is printed +%%%

Reason, the reason for aborting the test case, is printed %%% in the test case log.

abort_current_testcase(Reason) -> test_server_ctrl:abort_current_testcase(Reason). @@ -999,13 +1006,13 @@ abort_current_testcase(Reason) -> %%% Reason = term() %%% %%% @doc

This function encrypts the source config file with DES3 and -%%% saves the result in file EncryptFileName. The key, +%%% saves the result in file EncryptFileName. The key, %%% a string, must be available in a text file named -%%% .ct_config.crypt in the current directory, or the +%%% .ct_config.crypt in the current directory, or the %%% home directory of the user (it is searched for in that order).

%%%

See the Common Test User's Guide for information about using %%% encrypted config files when running tests.

-%%%

See the crypto application for details on DES3 +%%%

See the crypto application for details on DES3 %%% encryption/decryption.

encrypt_config_file(SrcFileName, EncryptFileName) -> ct_config:encrypt_config_file(SrcFileName, EncryptFileName). @@ -1019,13 +1026,13 @@ encrypt_config_file(SrcFileName, EncryptFileName) -> %%% Reason = term() %%% %%% @doc

This function encrypts the source config file with DES3 and -%%% saves the result in the target file EncryptFileName. +%%% saves the result in the target file EncryptFileName. %%% The encryption key to use is either the value in -%%% {key,Key} or the value stored in the file specified -%%% by {file,File}.

+%%% {key,Key} or the value stored in the file specified +%%% by {file,File}.

%%%

See the Common Test User's Guide for information about using %%% encrypted config files when running tests.

-%%%

See the crypto application for details on DES3 +%%%

See the crypto application for details on DES3 %%% encryption/decryption.

encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> ct_config:encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile). @@ -1037,11 +1044,11 @@ encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> %%% TargetFileName = string() %%% Reason = term() %%% -%%% @doc

This function decrypts EncryptFileName, previously -%%% generated with encrypt_config_file/2/3. The original +%%% @doc

This function decrypts EncryptFileName, previously +%%% generated with encrypt_config_file/2/3. The original %%% file contents is saved in the target file. The encryption key, a %%% string, must be available in a text file named -%%% .ct_config.crypt in the current directory, or the +%%% .ct_config.crypt in the current directory, or the %%% home directory of the user (it is searched for in that order).

decrypt_config_file(EncryptFileName, TargetFileName) -> ct_config:decrypt_config_file(EncryptFileName, TargetFileName). @@ -1054,8 +1061,8 @@ decrypt_config_file(EncryptFileName, TargetFileName) -> %%% KeyOrFile = {key,string()} | {file,string()} %%% Reason = term() %%% -%%% @doc

This function decrypts EncryptFileName, previously -%%% generated with encrypt_config_file/2/3. The original +%%% @doc

This function decrypts EncryptFileName, previously +%%% generated with encrypt_config_file/2/3. The original %%% file contents is saved in the target file. The key must have the %%% the same value as that used for encryption.

decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) -> @@ -1072,7 +1079,7 @@ decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) -> %%% given callback module and configuration string. Callback module %%% should be either loaded or present in the code part. Loaded %%% configuration variables can later be removed using -%%% remove_config/2 function.

+%%% remove_config/2 function.

add_config(Callback, Config)-> ct_config:add_config(Callback, Config). @@ -1101,9 +1108,9 @@ remove_config(Callback, Config) -> %%% A = list() %%% %%% @doc

Use this function to set a new timetrap for the running test case. -%%% If the argument is Func, the timetrap will be triggered -%%% when this function returns. Func may also return a new -%%% Time value, which in that case will be the value for the +%%% If the argument is Func, the timetrap will be triggered +%%% when this function returns. Func may also return a new +%%% Time value, which in that case will be the value for the %%% new timetrap.

timetrap(Time) -> test_server:timetrap_cancel(), @@ -1174,13 +1181,21 @@ sync_notify(Name,Data) -> %%% 'enable break with release_shell option' %%% TestCases = [atom()] %%% -%%% @doc

This function will cancel all timetraps and pause the +%%% @doc

This function will cancel any active timetrap and pause the %%% execution of the current test case until the user calls the %%% continue/0 function. It gives the user the opportunity %%% to interact with the erlang node running the tests, e.g. for %%% debugging purposes or for manually executing a part of the %%% test case. If a parallel group is executing, break/2 %%% should be called instead.

+%%%

A cancelled timetrap will not be automatically +%%% reactivated after the break, but must be started exlicitly with +%%% ct:timetrap/1

+%%%

In order for the break/continue functionality to work, +%%% Common Test must release the shell process controlling stdin. +%%% This is done by setting the release_shell start option +%%% to true. See the User's Guide for more information.

+ break(Comment) -> case {ct_util:get_testdata(starter), ct_util:get_testdata(release_shell)} of @@ -1191,7 +1206,7 @@ break(Comment) -> {error,'enable break with release_shell option'}; _ -> case get_testdata(curr_tc) of - {ok,{_,TestCase}} -> + {ok,{_,_TestCase}} -> test_server:break(?MODULE, Comment); {ok,Cases} when is_list(Cases) -> {error,{'multiple cases running', @@ -1215,6 +1230,7 @@ break(Comment) -> %%% pause a test case executing in a parallel group. The %%% continue/1 function should be used to resume %%% execution of TestCase.

+%%%

See break/1 for more details.

break(TestCase, Comment) -> case {ct_util:get_testdata(starter), ct_util:get_testdata(release_shell)} of diff --git a/lib/test_server/doc/src/test_server_ctrl.xml b/lib/test_server/doc/src/test_server_ctrl.xml index 9028a67ecb..6b33591701 100644 --- a/lib/test_server/doc/src/test_server_ctrl.xml +++ b/lib/test_server/doc/src/test_server_ctrl.xml @@ -769,11 +769,13 @@ Optional, if not given the test server controller node constantly updated. The following can be reported:

What = tests_start, Data = {Name,NumCases}

+ What = loginfo, Data = [{topdir,TestRootDir},{rundir,CurrLogDir}]

What = tests_done, Data = {Ok,Failed,{UserSkipped,AutoSkipped}}

- What = tc_start, Data = {Mod,Func}

+ What = tc_start, Data = {{Mod,Func},TCLogFile}

What = tc_done, Data = {Mod,Func,Result}

What = tc_user_skip, Data = {Mod,Func,Comment}

- What = tc_auto_skip, Data = {Mod,Func,Comment}

+ What = tc_auto_skip, Data = {Mod,Func,Comment}

+ What = framework_error, Data = {{FWMod,FWFunc},Error}

-- cgit v1.2.3