aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/doc/src/ct_run.xml1
-rw-r--r--lib/common_test/doc/src/run_test_chapter.xml817
-rw-r--r--lib/common_test/src/ct.erl4
-rw-r--r--lib/common_test/src/ct_config.erl5
-rw-r--r--lib/common_test/src/ct_master.erl155
-rw-r--r--lib/common_test/src/ct_run.erl524
-rw-r--r--lib/common_test/src/ct_testspec.erl320
-rw-r--r--lib/common_test/test/Makefile1
-rw-r--r--lib/common_test/test/ct_master_SUITE.erl14
-rw-r--r--lib/common_test/test/ct_test_support.erl2
-rw-r--r--lib/common_test/test/ct_testspec_2_SUITE.erl20
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE.erl749
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg111
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg121
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg131
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg211
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec14
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep15
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join16
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join11
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep13
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both24
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join29
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl175
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl175
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl174
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl177
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl158
39 files changed, 2763 insertions, 787 deletions
diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml
index 0750f560b3..198290c1be 100644
--- a/lib/common_test/doc/src/ct_run.xml
+++ b/lib/common_test/doc/src/ct_run.xml
@@ -126,6 +126,7 @@
<title>Run tests using test specification</title>
<pre>
ct_run -spec TestSpec1 TestSpec2 .. TestSpecN
+ [-join_specs]
[-config ConfigFile1 ConfigFile2 .. ConfigFileN]
[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
ConfigString2 and .. and CallbackModuleN ConfigStringN]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index d5f5d89e05..35f89153d3 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -530,374 +530,469 @@
<marker id="test_specifications"></marker>
<section>
<title>Test Specifications</title>
-
- <p>The most flexible way to specify what to test, is to use a so
- called test specification. A test specification is a sequence of
- Erlang terms. The terms are normally declared in a text file (see
- <c><seealso marker="ct#run_test-1">ct:run_test/1</seealso></c>), but
- may also be passed to Common Test on the form of a list (see
- <c><seealso marker="ct#run_testspec-1">ct:run_testspec/1</seealso></c>).
- There are two general types of terms: configuration terms and test
- specification terms.</p>
- <p>With configuration terms it is possible to e.g. label the test
- run (similar to <c>ct_run -label</c>), evaluate arbitrary expressions
- before starting the test, import configuration data (similar to
- <c>ct_run -config/-userconfig</c>), specify the top level HTML log
- directory (similar to <c>ct_run -logdir</c>), enable code coverage
- analysis (similar to <c>ct_run -cover</c>), install Common Test Hooks
- (similar to <c>ct_run -ch_hooks</c>), install event_handler plugins
- (similar to <c>ct_run -event_handler</c>), specify include directories
- that should be passed to the compiler for automatic compilation
- (similar to <c>ct_run -include</c>), disable the auto compilation
- feature (similar to <c>ct_run -no_auto_compile</c>), set verbosity
- levels (similar to <c>ct_run -verbosity</c>), and more.</p>
- <p>Configuration terms can be combined with <c>ct_run</c> start flags,
- or <c>ct:run_test/1</c> options. The result will for some flags/options
- and terms be that the values are merged (e.g. configuration files,
- include directories, verbosity levels, silent connections), and for
- others that the start flags/options override the test specification
- terms (e.g. log directory, label, style sheet, auto compilation).</p>
- <p>With test specification terms it is possible to state exactly
- which tests should run and in which order. A test term specifies
- either one or more suites, one or more test case groups (possibly nested),
- or one or more test cases in a group (or in multiple groups) or in a suite.</p>
- <p>An arbitrary number of test terms may be declared in sequence.
- Common Test will by default compile the terms into one or more tests
- to be performed in one resulting test run. Note that a term that
- specifies a set of test cases will "swallow" one that only
- specifies a subset of these cases. E.g. the result of merging
- one term that specifies that all cases in suite S should be
- executed, with another term specifying only test case X and Y in
- S, is a test of all cases in S. However, if a term specifying
- test case X and Y in S is merged with a term specifying case Z
- in S, the result is a test of X, Y and Z in S. To disable this
- behaviour, i.e. to instead perform each test sequentially in a "script-like"
- manner, the term <c>merge_tests</c> can be set to <c>false</c> in
- the test specification.</p>
- <p>A test term can also specify one or more test suites, groups,
- or test cases to be skipped. Skipped suites, groups and cases
- are not executed and show up in the HTML log files as
- SKIPPED.</p>
- <p>When a test case group is specified, the resulting test
- executes the <c>init_per_group</c> function, followed by all test
- cases and sub groups (including their configuration functions), and
- finally the <c>end_per_group</c> function. Also if particular
- test cases in a group are specified, <c>init_per_group</c>
- and <c>end_per_group</c> for the group in question are
- called. If a group which is defined (in <c>Suite:group/0</c>) to
- be a sub group of another group, is specified (or if particular test
- cases of a sub group are), Common Test will call the configuration
- functions for the top level groups as well as for the sub group
- in question (making it possible to pass configuration data all
- the way from <c>init_per_suite</c> down to the test cases in the
- sub group).</p>
- <p>The test specification utilizes the same mechanism for specifying
- test case groups by means of names and paths, as explained in the
- <seealso marker="run_test_chapter#group_execution">Group Execution</seealso>
- section above, with the addition of the <c>GroupSpec</c> element
- described next.</p>
- <p>The <c>GroupSpec</c> element makes it possible to specify
- group execution properties that will override those in the
- group definition (i.e. in <c>groups/0</c>). Execution properties for
- sub-groups may be overridden as well. This feature makes it possible to
- change properties of groups at the time of execution,
- without even having to edit the test suite. The very same
- feature is available for <c>group</c> elements in the <c>Suite:all/0</c>
- list. Therefore, more detailed documentation, and examples, can be
- found in the <seealso marker="write_test_chapter#test_case_groups">
- Test case groups</seealso> chapter.</p>
-
- <p>Below is the test specification syntax. Test specifications can
- be used to run tests both in a single test host environment and
- in a distributed Common Test environment (Large Scale
- Testing). The node parameters in the <c>init</c> term are only
- relevant in the latter (see the
- <seealso marker="ct_master_chapter#test_specifications">Large
- Scale Testing</seealso> chapter for information). For more information
- about the various terms, please see the corresponding sections in the
- User's Guide, such as e.g. the
- <seealso marker="run_test_chapter#ct_run"><c>ct_run</c>
- program</seealso> for an overview of available start flags
- (since most flags have a corresponding configuration term), and
- more detailed explanation of e.g.
- <seealso marker="write_test_chapter#logging">Logging</seealso>
- (for the <c>verbosity</c>, <c>stylesheet</c> and <c>basic_html</c> terms),
- <seealso marker="config_file_chapter#top">External Configuration Data</seealso>
- (for the <c>config</c> and <c>userconfig</c> terms),
- <seealso marker="event_handler_chapter#event_handling">Event
- Handling</seealso> (for the <c>event_handler</c> term),
- <seealso marker="ct_hooks_chapter#installing">Common Test Hooks</seealso>
- (for the <c>ct_hooks</c> term), etc.</p>
- <p>Config terms:</p>
- <pre>
- {merge_tests, Bool}.
-
- {define, Constant, Value}.
-
- {node, NodeAlias, Node}.
-
- {init, InitOptions}.
- {init, [NodeAlias], InitOptions}.
-
- {label, Label}.
- {label, NodeRefs, Label}.
-
- {verbosity, VerbosityLevels}.
- {verbosity, NodeRefs, VerbosityLevels}.
-
- {stylesheet, CSSFile}.
- {stylesheet, NodeRefs, CSSFile}.
-
- {silent_connections, ConnTypes}.
- {silent_connections, NodeRefs, ConnTypes}.
-
- {multiply_timetraps, N}.
- {multiply_timetraps, NodeRefs, N}.
-
- {scale_timetraps, Bool}.
- {scale_timetraps, NodeRefs, Bool}.
-
- {cover, CoverSpecFile}.
- {cover, NodeRefs, CoverSpecFile}.
-
- {cover_stop, Bool}.
- {cover_stop, NodeRefs, Bool}.
-
- {include, IncludeDirs}.
- {include, NodeRefs, IncludeDirs}.
-
- {auto_compile, Bool},
- {auto_compile, NodeRefs, Bool},
-
- {config, ConfigFiles}.
- {config, ConfigDir, ConfigBaseNames}.
- {config, NodeRefs, ConfigFiles}.
- {config, NodeRefs, ConfigDir, ConfigBaseNames}.
-
- {userconfig, {CallbackModule, ConfigStrings}}.
- {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}.
-
- {logdir, LogDir}.
- {logdir, NodeRefs, LogDir}.
-
- {logopts, LogOpts}.
- {logopts, NodeRefs, LogOpts}.
-
- {create_priv_dir, PrivDirOption}.
- {create_priv_dir, NodeRefs, PrivDirOption}.
-
- {event_handler, EventHandlers}.
- {event_handler, NodeRefs, EventHandlers}.
- {event_handler, EventHandlers, InitArgs}.
- {event_handler, NodeRefs, EventHandlers, InitArgs}.
-
- {ct_hooks, CTHModules}.
- {ct_hooks, NodeRefs, CTHModules}.
-
- {enable_builtin_hooks, Bool}.
-
- {basic_html, Bool}.
- {basic_html, NodeRefs, Bool}.
+ <section>
+ <title>General description</title>
+ <p>The most flexible way to specify what to test, is to use a so
+ called test specification. A test specification is a sequence of
+ Erlang terms. The terms are normally declared in one or more text files
+ (see <c><seealso marker="ct#run_test-1">ct:run_test/1</seealso></c>), but
+ may also be passed to Common Test on the form of a list (see
+ <c><seealso marker="ct#run_testspec-1">ct:run_testspec/1</seealso></c>).
+ There are two general types of terms: configuration terms and test
+ specification terms.</p>
+ <p>With configuration terms it is possible to e.g. label the test
+ run (similar to <c>ct_run -label</c>), evaluate arbitrary expressions
+ before starting the test, import configuration data (similar to
+ <c>ct_run -config/-userconfig</c>), specify the top level HTML log
+ directory (similar to <c>ct_run -logdir</c>), enable code coverage
+ analysis (similar to <c>ct_run -cover</c>), install Common Test Hooks
+ (similar to <c>ct_run -ch_hooks</c>), install event_handler plugins
+ (similar to <c>ct_run -event_handler</c>), specify include directories
+ that should be passed to the compiler for automatic compilation
+ (similar to <c>ct_run -include</c>), disable the auto compilation
+ feature (similar to <c>ct_run -no_auto_compile</c>), set verbosity
+ levels (similar to <c>ct_run -verbosity</c>), and more.</p>
+ <p>Configuration terms can be combined with <c>ct_run</c> start flags,
+ or <c>ct:run_test/1</c> options. The result will for some flags/options
+ and terms be that the values are merged (e.g. configuration files,
+ include directories, verbosity levels, silent connections), and for
+ others that the start flags/options override the test specification
+ terms (e.g. log directory, label, style sheet, auto compilation).</p>
+ <p>With test specification terms it is possible to state exactly
+ which tests should run and in which order. A test term specifies
+ either one or more suites, one or more test case groups (possibly nested),
+ or one or more test cases in a group (or in multiple groups) or in a suite.</p>
+ <p>An arbitrary number of test terms may be declared in sequence.
+ Common Test will by default compile the terms into one or more tests
+ to be performed in one resulting test run. Note that a term that
+ specifies a set of test cases will "swallow" one that only
+ specifies a subset of these cases. E.g. the result of merging
+ one term that specifies that all cases in suite S should be
+ executed, with another term specifying only test case X and Y in
+ S, is a test of all cases in S. However, if a term specifying
+ test case X and Y in S is merged with a term specifying case Z
+ in S, the result is a test of X, Y and Z in S. To disable this
+ behaviour, i.e. to instead perform each test sequentially in a "script-like"
+ manner, the term <c>merge_tests</c> can be set to <c>false</c> in
+ the test specification.</p>
+ <p>A test term can also specify one or more test suites, groups,
+ or test cases to be skipped. Skipped suites, groups and cases
+ are not executed and show up in the HTML log files as
+ SKIPPED.</p>
+ </section>
+ <section>
+ <title>Using multiple test specification files</title>
+
+ <p>When multiple test specification files are given at startup (either
+ with <c>ct_run -spec file1 file2 ...</c> or
+ <c>ct:run_test([{spec, [File1,File2,...]}])</c>),
+ Common Test will either execute one test run per specification file, or
+ join the files and perform all tests within one single test run. The first
+ behaviour is the default one. The latter requires that the start
+ flag/option <c>join_suites</c> is provided, e.g.
+ <c>run_test -spec ./my_tests1.ts ./my_tests2.ts -join_suites</c>.</p>
+
+ <p>Joining a number of specifications, or running them separately, can
+ also be accomplished with (and may be combined with) test specification
+ file inclusion, described next.</p>
+ </section>
+ <section>
+ <title>Test specification file inclusion</title>
+ <p>With the <c>specs</c> term (see syntax below), it's possible to have
+ a test specification include other specifications. An included
+ specification may either be joined with the source specification,
+ or used to produce a separate test run (like with the <c>join_specs</c>
+ start flag/option above). Example:</p>
+ <pre>
+ %% In specification file "a.spec"
+ {specs, join, ["b.spec", "c.spec"]}.
+ {specs, separate, ["d.spec", "e.spec"]}.
+ %% Config and test terms follow
+ ...</pre>
+ <p>In this example, the test terms defined in files "b.spec" and "c.spec"
+ will be joined with the terms in the source specification "a.spec"
+ (if any). The inclusion of specifications "d.spec" and
+ "e.spec" will result in two separate, and independent, test runs (i.e.
+ one for each included specification).</p>
+ <p>Note that the <c>join</c> option does not imply that the test terms
+ will be merged (see <c>merge_tests</c> above), only that all tests are
+ executed in one single test run.</p>
+ <p>Joined specifications share common configuration settings, such as
+ the list of <c>config</c> files or <c>include</c> directories.
+ For configuration that can not be combined, such as settings for <c>logdir</c>
+ or <c>verbosity</c>, it is up to the user to ensure there are no clashes
+ when the test specifications are joined. Specifications included with
+ the <c>separate</c> option, do not share configuration settings with the
+ source specification. This is useful e.g. if there are clashing
+ configuration settings in included specifications, making it impossible
+ to join them.</p>
+ <p>If <c>{merge_tests,true}</c> is set in the source specification
+ (which is the default setting), terms in joined specifications will be
+ merged with terms in the source specification (according to the
+ description of <c>merge_tests</c> above).</p>
+ <p>Note that it is always the <c>merge_tests</c> setting in the source
+ specification that is used when joined with other specifications.
+ Say e.g. that a source specification A, with tests TA1 and TA2, has
+ <c>{merge_tests,false}</c> set, and it includes another specification,
+ B, with tests TB1 and TB2, that has <c>{merge_tests,true}</c> set.
+ The result will be that the test series: <c>TA1,TA2,merge(TB1,TB2)</c>,
+ is executed. The opposite <c>merge_tests</c> settings would result in the
+ following the test series: <c>merge(merge(TA1,TA2),TB1,TB2)</c>.</p>
+ <p>The <c>specs</c> term may of course be used to nest specifications,
+ i.e. have one specification include other specifications, which in turn
+ include others, etc.</p>
+ </section>
+ <section>
+ <title>Test case groups</title>
+
+ <p>When a test case group is specified, the resulting test
+ executes the <c>init_per_group</c> function, followed by all test
+ cases and sub groups (including their configuration functions), and
+ finally the <c>end_per_group</c> function. Also if particular
+ test cases in a group are specified, <c>init_per_group</c>
+ and <c>end_per_group</c> for the group in question are
+ called. If a group which is defined (in <c>Suite:group/0</c>) to
+ be a sub group of another group, is specified (or if particular test
+ cases of a sub group are), Common Test will call the configuration
+ functions for the top level groups as well as for the sub group
+ in question (making it possible to pass configuration data all
+ the way from <c>init_per_suite</c> down to the test cases in the
+ sub group).</p>
+ <p>The test specification utilizes the same mechanism for specifying
+ test case groups by means of names and paths, as explained in the
+ <seealso marker="run_test_chapter#group_execution">Group Execution</seealso>
+ section above, with the addition of the <c>GroupSpec</c> element
+ described next.</p>
+ <p>The <c>GroupSpec</c> element makes it possible to specify
+ group execution properties that will override those in the
+ group definition (i.e. in <c>groups/0</c>). Execution properties for
+ sub-groups may be overridden as well. This feature makes it possible to
+ change properties of groups at the time of execution,
+ without even having to edit the test suite. The very same
+ feature is available for <c>group</c> elements in the <c>Suite:all/0</c>
+ list. Therefore, more detailed documentation, and examples, can be
+ found in the <seealso marker="write_test_chapter#test_case_groups">
+ Test case groups</seealso> chapter.</p>
+ </section>
- {release_shell, Bool}.</pre>
+ <section>
+ <title>Test specification syntax</title>
+
+ <p>Below is the test specification syntax. Test specifications can
+ be used to run tests both in a single test host environment and
+ in a distributed Common Test environment (Large Scale
+ Testing). The node parameters in the <c>init</c> term are only
+ relevant in the latter (see the
+ <seealso marker="ct_master_chapter#test_specifications">Large
+ Scale Testing</seealso> chapter for information). For more information
+ about the various terms, please see the corresponding sections in the
+ User's Guide, such as e.g. the
+ <seealso marker="run_test_chapter#ct_run"><c>ct_run</c>
+ program</seealso> for an overview of available start flags
+ (since most flags have a corresponding configuration term), and
+ more detailed explanation of e.g.
+ <seealso marker="write_test_chapter#logging">Logging</seealso>
+ (for the <c>verbosity</c>, <c>stylesheet</c> and <c>basic_html</c> terms),
+ <seealso marker="config_file_chapter#top">External Configuration Data</seealso>
+ (for the <c>config</c> and <c>userconfig</c> terms),
+ <seealso marker="event_handler_chapter#event_handling">Event
+ Handling</seealso> (for the <c>event_handler</c> term),
+ <seealso marker="ct_hooks_chapter#installing">Common Test Hooks</seealso>
+ (for the <c>ct_hooks</c> term), etc.</p>
+ </section>
+ <p>Config terms:</p>
+ <pre>
+ {merge_tests, Bool}.
+
+ {define, Constant, Value}.
+
+ {specs, InclSpecsOption, TestSpecs}.
+
+ {node, NodeAlias, Node}.
+
+ {init, InitOptions}.
+ {init, [NodeAlias], InitOptions}.
+
+ {label, Label}.
+ {label, NodeRefs, Label}.
+
+ {verbosity, VerbosityLevels}.
+ {verbosity, NodeRefs, VerbosityLevels}.
+
+ {stylesheet, CSSFile}.
+ {stylesheet, NodeRefs, CSSFile}.
+
+ {silent_connections, ConnTypes}.
+ {silent_connections, NodeRefs, ConnTypes}.
+
+ {multiply_timetraps, N}.
+ {multiply_timetraps, NodeRefs, N}.
+
+ {scale_timetraps, Bool}.
+ {scale_timetraps, NodeRefs, Bool}.
+
+ {cover, CoverSpecFile}.
+ {cover, NodeRefs, CoverSpecFile}.
+
+ {cover_stop, Bool}.
+ {cover_stop, NodeRefs, Bool}.
+
+ {include, IncludeDirs}.
+ {include, NodeRefs, IncludeDirs}.
+
+ {auto_compile, Bool},
+ {auto_compile, NodeRefs, Bool},
+
+ {config, ConfigFiles}.
+ {config, ConfigDir, ConfigBaseNames}.
+ {config, NodeRefs, ConfigFiles}.
+ {config, NodeRefs, ConfigDir, ConfigBaseNames}.
+
+ {userconfig, {CallbackModule, ConfigStrings}}.
+ {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}.
+
+ {logdir, LogDir}.
+ {logdir, NodeRefs, LogDir}.
+
+ {logopts, LogOpts}.
+ {logopts, NodeRefs, LogOpts}.
+
+ {create_priv_dir, PrivDirOption}.
+ {create_priv_dir, NodeRefs, PrivDirOption}.
+
+ {event_handler, EventHandlers}.
+ {event_handler, NodeRefs, EventHandlers}.
+ {event_handler, EventHandlers, InitArgs}.
+ {event_handler, NodeRefs, EventHandlers, InitArgs}.
+
+ {ct_hooks, CTHModules}.
+ {ct_hooks, NodeRefs, CTHModules}.
+
+ {enable_builtin_hooks, Bool}.
+
+ {basic_html, Bool}.
+ {basic_html, NodeRefs, Bool}.
+
+ {release_shell, Bool}.</pre>
+
<p>Test terms:</p>
- <pre>
- {suites, Dir, Suites}.
- {suites, NodeRefs, Dir, Suites}.
-
- {groups, Dir, Suite, Groups}.
- {groups, NodeRefs, Dir, Suite, Groups}.
-
- {groups, Dir, Suite, Groups, {cases,Cases}}.
- {groups, NodeRefs, Dir, Suite, Groups, {cases,Cases}}.
-
- {cases, Dir, Suite, Cases}.
- {cases, NodeRefs, Dir, Suite, Cases}.
-
- {skip_suites, Dir, Suites, Comment}.
- {skip_suites, NodeRefs, Dir, Suites, Comment}.
-
- {skip_groups, Dir, Suite, GroupNames, Comment}.
- {skip_groups, NodeRefs, Dir, Suite, GroupNames, Comment}.
-
- {skip_cases, Dir, Suite, Cases, Comment}.
- {skip_cases, NodeRefs, Dir, Suite, Cases, Comment}.</pre>
-
+ <pre>
+ {suites, Dir, Suites}.
+ {suites, NodeRefs, Dir, Suites}.
+
+ {groups, Dir, Suite, Groups}.
+ {groups, NodeRefs, Dir, Suite, Groups}.
+
+ {groups, Dir, Suite, Groups, {cases,Cases}}.
+ {groups, NodeRefs, Dir, Suite, Groups, {cases,Cases}}.
+
+ {cases, Dir, Suite, Cases}.
+ {cases, NodeRefs, Dir, Suite, Cases}.
+
+ {skip_suites, Dir, Suites, Comment}.
+ {skip_suites, NodeRefs, Dir, Suites, Comment}.
+
+ {skip_groups, Dir, Suite, GroupNames, Comment}.
+ {skip_groups, NodeRefs, Dir, Suite, GroupNames, Comment}.
+
+ {skip_cases, Dir, Suite, Cases, Comment}.
+ {skip_cases, NodeRefs, Dir, Suite, Cases, Comment}.</pre>
+
<p>Types:</p>
- <pre>
- Bool = true | false
- Constant = atom()
- Value = term()
- NodeAlias = atom()
- Node = node()
- NodeRef = NodeAlias | Node | master
- NodeRefs = all_nodes | [NodeRef] | NodeRef
- InitOptions = term()
- Label = atom() | string()
- VerbosityLevels = integer() | [{Category,integer()}]
- Category = atom()
- CSSFile = string()
- ConnTypes = all | [atom()]
- N = integer()
- CoverSpecFile = string()
- IncludeDirs = string() | [string()]
- ConfigFiles = string() | [string()]
- ConfigDir = string()
- ConfigBaseNames = string() | [string()]
- CallbackModule = atom()
- ConfigStrings = string() | [string()]
- LogDir = string()
- LogOpts = [term()]
- PrivDirOption = auto_per_run | auto_per_tc | manual_per_tc
- EventHandlers = atom() | [atom()]
- InitArgs = [term()]
- CTHModules = [CTHModule |
- {CTHModule, CTHInitArgs} |
- {CTHModule, CTHInitArgs, CTHPriority}]
- CTHModule = atom()
- CTHInitArgs = term()
- Dir = string()
- Suites = atom() | [atom()] | all
- Suite = atom()
- Groups = GroupPath | [GroupPath] | GroupSpec | [GroupSpec] | all
- GroupPath = [GroupName]
- GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec}
- GroupName = atom()
- GroupNames = GroupName | [GroupName]
- Cases = atom() | [atom()] | all
- Comment = string() | ""</pre>
-
- <p>The difference between the <c>config</c> terms above, is that with
- <c>ConfigDir</c>, <c>ConfigBaseNames</c> is a list of base names,
- i.e. without directory paths. <c>ConfigFiles</c> must be full names,
- including paths. E.g, these two terms have the same meaning:</p>
- <pre>
- {config, ["/home/testuser/tests/config/nodeA.cfg",
- "/home/testuser/tests/config/nodeB.cfg"]}.
-
- {config, "/home/testuser/tests/config", ["nodeA.cfg","nodeB.cfg"]}.</pre>
-
- <note><p>Any relative paths specified in the test specification, will be
- relative to the directory which contains the test specification file, if
- <c>ct_run -spec TestSpecFile ...</c> or
- <c>ct:run:test([{spec,TestSpecFile},...])</c>
- executes the test. The path will be relative to the top level log directory, if
- <c>ct:run:testspec(TestSpec)</c> executes the test.</p></note>
-
- <p>The <c>define</c> term introduces a constant, which is used to
- replace the name <c>Constant</c> with <c>Value</c>, wherever it's found in
- the test specification. This replacement happens during an initial iteration
- through the test specification. Constants may be used anywhere in the test
- specification, e.g. in arbitrary lists and tuples, and even in strings
- and inside the value part of other constant definitions! A constant can
- also be part of a node name, but that is the only place where a constant
- can be part of an atom.</p>
-
- <note><p>For the sake of readability, the name of the constant must always
- begin with an upper case letter, or a <c>$</c>, <c>?</c>, or <c>_</c>.
- This also means that it must always be single quoted (obviously, since
- the constant name is actually an atom, not text).</p></note>
-
- <p>The main benefit of constants is that they can be used to reduce the size
- (and avoid repetition) of long strings, such as file paths. Compare these
- terms:</p>
-
- <pre>
- %% 1a. no constant
- {config, "/home/testuser/tests/config", ["nodeA.cfg","nodeB.cfg"]}.
- {suites, "/home/testuser/tests/suites", all}.
-
- %% 1b. with constant
- {define, 'TESTDIR', "/home/testuser/tests"}.
- {config, "'TESTDIR'/config", ["nodeA.cfg","nodeB.cfg"]}.
- {suites, "'TESTDIR'/suites", all}.
-
- %% 2a. no constants
- {config, [testnode@host1, testnode@host2], "../config", ["nodeA.cfg","nodeB.cfg"]}.
- {suites, [testnode@host1, testnode@host2], "../suites", [x_SUITE, y_SUITE]}.
-
- %% 2b. with constants
- {define, 'NODE', testnode}.
- {define, 'NODES', ['NODE'@host1, 'NODE'@host2]}.
- {config, 'NODES', "../config", ["nodeA.cfg","nodeB.cfg"]}.
- {suites, 'NODES', "../suites", [x_SUITE, y_SUITE]}.</pre>
-
- <p>Constants make the test specification term <c>alias</c>, in previous
- versions of Common Test, redundant. This term has been deprecated but will
- remain supported in upcoming Common Test releases. Replacing <c>alias</c>
- terms with <c>define</c> is strongly recommended though! Here's an example
- of such a replacement:</p>
+ <pre>
+ Bool = true | false
+ Constant = atom()
+ Value = term()
+ InclSpecsOption = join | separate
+ TestSpecs = string() | [string()]
+ NodeAlias = atom()
+ Node = node()
+ NodeRef = NodeAlias | Node | master
+ NodeRefs = all_nodes | [NodeRef] | NodeRef
+ InitOptions = term()
+ Label = atom() | string()
+ VerbosityLevels = integer() | [{Category,integer()}]
+ Category = atom()
+ CSSFile = string()
+ ConnTypes = all | [atom()]
+ N = integer()
+ CoverSpecFile = string()
+ IncludeDirs = string() | [string()]
+ ConfigFiles = string() | [string()]
+ ConfigDir = string()
+ ConfigBaseNames = string() | [string()]
+ CallbackModule = atom()
+ ConfigStrings = string() | [string()]
+ LogDir = string()
+ LogOpts = [term()]
+ PrivDirOption = auto_per_run | auto_per_tc | manual_per_tc
+ EventHandlers = atom() | [atom()]
+ InitArgs = [term()]
+ CTHModules = [CTHModule |
+ {CTHModule, CTHInitArgs} |
+ {CTHModule, CTHInitArgs, CTHPriority}]
+ CTHModule = atom()
+ CTHInitArgs = term()
+ Dir = string()
+ Suites = atom() | [atom()] | all
+ Suite = atom()
+ Groups = GroupPath | [GroupPath] | GroupSpec | [GroupSpec] | all
+ GroupPath = [GroupName]
+ GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec}
+ GroupName = atom()
+ GroupNames = GroupName | [GroupName]
+ Cases = atom() | [atom()] | all
+ Comment = string() | ""</pre>
+
+ <section>
+ <p>The difference between the <c>config</c> terms above, is that with
+ <c>ConfigDir</c>, <c>ConfigBaseNames</c> is a list of base names,
+ i.e. without directory paths. <c>ConfigFiles</c> must be full names,
+ including paths. E.g, these two terms have the same meaning:</p>
+ <pre>
+ {config, ["/home/testuser/tests/config/nodeA.cfg",
+ "/home/testuser/tests/config/nodeB.cfg"]}.
+
+ {config, "/home/testuser/tests/config", ["nodeA.cfg","nodeB.cfg"]}.</pre>
+
+ <note><p>Any relative paths specified in the test specification, will be
+ relative to the directory which contains the test specification file, if
+ <c>ct_run -spec TestSpecFile ...</c> or
+ <c>ct:run:test([{spec,TestSpecFile},...])</c>
+ executes the test. The path will be relative to the top level log directory, if
+ <c>ct:run:testspec(TestSpec)</c> executes the test.</p></note>
+ </section>
- <pre>
- %% using the old alias term
- {config, "/home/testuser/tests/config/nodeA.cfg"}.
- {alias, suite_dir, "/home/testuser/tests/suites"}.
- {groups, suite_dir, x_SUITE, group1}.
-
- %% replacing with constants
- {define, 'TestDir', "/home/testuser/tests"}.
- {define, 'CfgDir', "'TestDir'/config"}.
- {define, 'SuiteDir', "'TestDir'/suites"}.
- {config, 'CfgDir', "nodeA.cfg"}.
- {groups, 'SuiteDir', x_SUITE, group1}.</pre>
-
- <p>Actually, constants could well replace the <c>node</c> term too, but
- this still has declarative value, mainly when used in combination
- with <c>NodeRefs == all_nodes</c> (see types above).</p>
-
- <p>Here follows a simple test specification example:</p>
- <pre>
- {define, 'Top', "/home/test"}.
- {define, 'T1', "'Top'/t1"}.
- {define, 'T2', "'Top'/t2"}.
- {define, 'T3', "'Top'/t3"}.
- {define, 'CfgFile', "config.cfg"}.
+ <section>
+ <title>Constants</title>
+
+ <p>The <c>define</c> term introduces a constant, which is used to
+ replace the name <c>Constant</c> with <c>Value</c>, wherever it's found in
+ the test specification. This replacement happens during an initial iteration
+ through the test specification. Constants may be used anywhere in the test
+ specification, e.g. in arbitrary lists and tuples, and even in strings
+ and inside the value part of other constant definitions! A constant can
+ also be part of a node name, but that is the only place where a constant
+ can be part of an atom.</p>
+
+ <note><p>For the sake of readability, the name of the constant must always
+ begin with an upper case letter, or a <c>$</c>, <c>?</c>, or <c>_</c>.
+ This also means that it must always be single quoted (obviously, since
+ the constant name is actually an atom, not text).</p></note>
+
+ <p>The main benefit of constants is that they can be used to reduce the size
+ (and avoid repetition) of long strings, such as file paths. Compare these
+ terms:</p>
+
+ <pre>
+ %% 1a. no constant
+ {config, "/home/testuser/tests/config", ["nodeA.cfg","nodeB.cfg"]}.
+ {suites, "/home/testuser/tests/suites", all}.
+
+ %% 1b. with constant
+ {define, 'TESTDIR', "/home/testuser/tests"}.
+ {config, "'TESTDIR'/config", ["nodeA.cfg","nodeB.cfg"]}.
+ {suites, "'TESTDIR'/suites", all}.
+
+ %% 2a. no constants
+ {config, [testnode@host1, testnode@host2], "../config", ["nodeA.cfg","nodeB.cfg"]}.
+ {suites, [testnode@host1, testnode@host2], "../suites", [x_SUITE, y_SUITE]}.
+
+ %% 2b. with constants
+ {define, 'NODE', testnode}.
+ {define, 'NODES', ['NODE'@host1, 'NODE'@host2]}.
+ {config, 'NODES', "../config", ["nodeA.cfg","nodeB.cfg"]}.
+ {suites, 'NODES', "../suites", [x_SUITE, y_SUITE]}.</pre>
+
+ <p>Constants make the test specification term <c>alias</c>, in previous
+ versions of Common Test, redundant. This term has been deprecated but will
+ remain supported in upcoming Common Test releases. Replacing <c>alias</c>
+ terms with <c>define</c> is strongly recommended though! Here's an example
+ of such a replacement:</p>
+
+ <pre>
+ %% using the old alias term
+ {config, "/home/testuser/tests/config/nodeA.cfg"}.
+ {alias, suite_dir, "/home/testuser/tests/suites"}.
+ {groups, suite_dir, x_SUITE, group1}.
+
+ %% replacing with constants
+ {define, 'TestDir', "/home/testuser/tests"}.
+ {define, 'CfgDir', "'TestDir'/config"}.
+ {define, 'SuiteDir', "'TestDir'/suites"}.
+ {config, 'CfgDir', "nodeA.cfg"}.
+ {groups, 'SuiteDir', x_SUITE, group1}.</pre>
+
+ <p>Actually, constants could well replace the <c>node</c> term too, but
+ this still has declarative value, mainly when used in combination
+ with <c>NodeRefs == all_nodes</c> (see types above).</p>
+ </section>
- {logdir, "'Top'/logs"}.
-
- {config, ["'T1'/'CfgFile'", "'T2'/'CfgFile'", "'T3'/'CfgFile'"]}.
-
- {suites, 'T1', all}.
- {skip_suites, 'T1', [t1B_SUITE,t1D_SUITE], "Not implemented"}.
- {skip_cases, 'T1', t1A_SUITE, [test3,test4], "Irrelevant"}.
- {skip_cases, 'T1', t1C_SUITE, [test1], "Ignore"}.
-
- {suites, 'T2', [t2B_SUITE,t2C_SUITE]}.
- {cases, 'T2', t2A_SUITE, [test4,test1,test7]}.
-
- {skip_suites, 'T3', all, "Not implemented"}.</pre>
+ <section>
+ <title>Example</title>
+
+ <p>Here follows a simple test specification example:</p>
+ <pre>
+ {define, 'Top', "/home/test"}.
+ {define, 'T1', "'Top'/t1"}.
+ {define, 'T2', "'Top'/t2"}.
+ {define, 'T3', "'Top'/t3"}.
+ {define, 'CfgFile', "config.cfg"}.
+
+ {logdir, "'Top'/logs"}.
+
+ {config, ["'T1'/'CfgFile'", "'T2'/'CfgFile'", "'T3'/'CfgFile'"]}.
+
+ {suites, 'T1', all}.
+ {skip_suites, 'T1', [t1B_SUITE,t1D_SUITE], "Not implemented"}.
+ {skip_cases, 'T1', t1A_SUITE, [test3,test4], "Irrelevant"}.
+ {skip_cases, 'T1', t1C_SUITE, [test1], "Ignore"}.
+
+ {suites, 'T2', [t2B_SUITE,t2C_SUITE]}.
+ {cases, 'T2', t2A_SUITE, [test4,test1,test7]}.
+
+ {skip_suites, 'T3', all, "Not implemented"}.</pre>
+
+ <p>The example specifies the following:</p>
+ <list>
+ <item>The specified logdir directory will be used for storing
+ the HTML log files (in subdirectories tagged with node name,
+ date and time).</item>
+ <item>The variables in the specified test system config files will be
+ imported for the test.</item>
+ <item>The first test to run includes all suites for system t1. Excluded from
+ the test are however the t1B and t1D suites. Also test cases test3 and
+ test4 in t1A as well as the test1 case in t1C are excluded from
+ the test.</item>
+ <item>Secondly, the test for system t2 should run. The included suites are
+ t2B and t2C. Included are also test cases test4, test1 and test7 in suite
+ t2A. Note that the test cases will be executed in the specified order.</item>
+ <item>Lastly, all suites for systems t3 are to be completely skipped and this
+ should be explicitly noted in the log files.</item>
+ </list>
+ </section>
- <p>The example specifies the following:</p>
- <list>
- <item>The specified logdir directory will be used for storing
- the HTML log files (in subdirectories tagged with node name,
- date and time).</item>
- <item>The variables in the specified test system config files will be
- imported for the test.</item>
- <item>The first test to run includes all suites for system t1. Excluded from
- the test are however the t1B and t1D suites. Also test cases test3 and
- test4 in t1A as well as the test1 case in t1C are excluded from
- the test.</item>
- <item>Secondly, the test for system t2 should run. The included suites are
- t2B and t2C. Included are also test cases test4, test1 and test7 in suite
- t2A. Note that the test cases will be executed in the specified order.</item>
- <item>Lastly, all suites for systems t3 are to be completely skipped and this
- should be explicitly noted in the log files.</item>
- </list>
- <p>With the <c>init</c> term it's possible to specify initialization options
- for nodes defined in the test specification. Currently, there are options
- to start the node and/or to evaluate any function on the node.
- See the <seealso marker="ct_master_chapter#ct_slave">Automatic startup of
- the test target nodes</seealso> chapter for details.</p>
- <p>It is possible for the user to provide a test specification that
- includes (for Common Test) unrecognizable terms. If this is desired,
- the <c>-allow_user_terms</c> flag should be used when starting tests with
- <c>ct_run</c>. This forces Common Test to ignore unrecognizable terms.
- Note that in this mode, Common Test is not able to check the specification
- for errors as efficiently as if the scanner runs in default mode.
- If <c><seealso marker="ct#run_test-1">ct:run_test/1</seealso></c> is used for starting the tests, the relaxed scanner
- mode is enabled by means of the tuple: <c>{allow_user_terms,true}</c></p>
+ <section>
+ <title>The init term</title>
+ <p>With the <c>init</c> term it's possible to specify initialization options
+ for nodes defined in the test specification. Currently, there are options
+ to start the node and/or to evaluate any function on the node.
+ See the <seealso marker="ct_master_chapter#ct_slave">Automatic startup of
+ the test target nodes</seealso> chapter for details.</p>
+ </section>
+ <section>
+ <title>User specific terms</title>
+ <p>It is possible for the user to provide a test specification that
+ includes (for Common Test) unrecognizable terms. If this is desired,
+ the <c>-allow_user_terms</c> flag should be used when starting tests with
+ <c>ct_run</c>. This forces Common Test to ignore unrecognizable terms.
+ Note that in this mode, Common Test is not able to check the specification
+ for errors as efficiently as if the scanner runs in default mode.
+ If <c><seealso marker="ct#run_test-1">ct:run_test/1</seealso></c> is used
+ for starting the tests, the relaxed scanner
+ mode is enabled by means of the tuple: <c>{allow_user_terms,true}</c></p>
+ </section>
</section>
<section>
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 1c7fafc491..04a95a53fa 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -144,8 +144,8 @@ run(TestDirs) ->
%%% @spec run_test(Opts) -> Result
%%% Opts = [OptTuples]
%%% OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} |
-%%% {testcase,Cases} | {spec,TestSpecs} | {label,Label} |
-%%% {config,CfgFiles} | {userconfig, UserConfig} |
+%%% {testcase,Cases} | {spec,TestSpecs} | {join_specs,Bool} |
+%%% {label,Label} | {config,CfgFiles} | {userconfig, UserConfig} |
%%% {allow_user_terms,Bool} | {logdir,LogDir} |
%%% {silent_connections,Conns} | {stylesheet,CSSFile} |
%%% {cover,CoverSpecFile} | {cover_stop,Bool} | {step,StepOpts} |
diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl
index ee5f868689..c35cbd3c08 100644
--- a/lib/common_test/src/ct_config.erl
+++ b/lib/common_test/src/ct_config.erl
@@ -266,7 +266,10 @@ read_config_files_int([{Callback, File}|Files], FunToSave) ->
read_config_files_int([], _FunToSave) ->
ok.
-store_config(Config, Callback, File) ->
+store_config(Config, Callback, File) when is_tuple(Config) ->
+ store_config([Config], Callback, File);
+
+store_config(Config, Callback, File) when is_list(Config) ->
[ets:insert(?attr_table,
#ct_conf{key=Key,
value=Val,
diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl
index 73ef15c500..e516f635d2 100644
--- a/lib/common_test/src/ct_master.erl
+++ b/lib/common_test/src/ct_master.erl
@@ -82,39 +82,48 @@ run_test(NodeOptsList) when is_list(NodeOptsList) ->
%%% ExclNodes = [atom()]
%%%
%%% @doc Tests are spawned on the nodes as specified in <code>TestSpecs</code>.
-%%% Each specification in TestSpec will be handled separately. It is however possible
-%%% to also specify a list of specifications that should be merged into one before
-%%% the tests are executed. Any test without a particular node specification will
-%%% also be executed on the nodes in <code>InclNodes</code>. Nodes in the
-%%% <code>ExclNodes</code> list will be excluded from the test.
+%%% Each specification in TestSpec will be handled separately. It is however
+%%% possible to also specify a list of specifications that should be merged
+%%% into one before the tests are executed. Any test without a particular node
+%%% specification will also be executed on the nodes in <code>InclNodes</code>.
+%%% Nodes in the <code>ExclNodes</code> list will be excluded from the test.
run([TS|TestSpecs],AllowUserTerms,InclNodes,ExclNodes) when is_list(TS),
is_list(InclNodes),
is_list(ExclNodes) ->
- TS1 =
- case TS of
- List=[S|_] when is_list(S) -> List;
- Spec -> [Spec]
- end,
- Result =
- case catch ct_testspec:collect_tests_from_file(TS1,InclNodes,AllowUserTerms) of
- {error,Reason} ->
- {error,Reason};
- TSRec=#testspec{logdir=AllLogDirs,
- config=StdCfgFiles,
- userconfig=UserCfgFiles,
- include=AllIncludes,
- init=AllInitOpts,
- event_handler=AllEvHs} ->
- AllCfgFiles = {StdCfgFiles, UserCfgFiles},
- RunSkipPerNode = ct_testspec:prepare_tests(TSRec),
- RunSkipPerNode2 = exclude_nodes(ExclNodes,RunSkipPerNode),
- run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs,
- AllIncludes,[],[],AllInitOpts,TS1)
- end,
- [{TS,Result} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)];
+ %% Note: [Spec] means run one test with Spec
+ %% [Spec1,Spec2] means run two tests separately
+ %% [[Spec1,Spec2]] means run one test, with the two specs merged
+ case catch ct_testspec:collect_tests_from_file([TS],InclNodes,
+ AllowUserTerms) of
+ {error,Reason} ->
+ [{error,Reason} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)];
+ Tests ->
+ RunResult =
+ lists:map(
+ fun({Specs,TSRec=#testspec{logdir=AllLogDirs,
+ config=StdCfgFiles,
+ userconfig=UserCfgFiles,
+ include=AllIncludes,
+ init=AllInitOpts,
+ event_handler=AllEvHs}}) ->
+ AllCfgFiles =
+ {StdCfgFiles,UserCfgFiles},
+ RunSkipPerNode =
+ ct_testspec:prepare_tests(TSRec),
+ RunSkipPerNode2 =
+ exclude_nodes(ExclNodes,RunSkipPerNode),
+ TSList = if is_integer(hd(TS)) -> [TS];
+ true -> TS end,
+ {Specs,run_all(RunSkipPerNode2,AllLogDirs,
+ AllCfgFiles,AllEvHs,
+ AllIncludes,[],[],AllInitOpts,TSList)}
+ end, Tests),
+ RunResult ++ run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)
+ end;
run([],_,_,_) ->
[];
-run(TS,AllowUserTerms,InclNodes,ExclNodes) when is_list(InclNodes), is_list(ExclNodes) ->
+run(TS,AllowUserTerms,InclNodes,ExclNodes) when is_list(InclNodes),
+ is_list(ExclNodes) ->
run([TS],AllowUserTerms,InclNodes,ExclNodes).
%%%-----------------------------------------------------------------
@@ -152,29 +161,32 @@ exclude_nodes([],RunSkipPerNode) ->
%%% AllowUserTerms = bool()
%%% Node = atom()
%%%
-%%% @doc Tests are spawned on <code>Node</code> according to <code>TestSpecs</code>.
+%%% @doc Tests are spawned on <code>Node</code> according to
+%%% <code>TestSpecs</code>.
run_on_node([TS|TestSpecs],AllowUserTerms,Node) when is_list(TS),is_atom(Node) ->
- TS1 =
- case TS of
- [List|_] when is_list(List) -> List;
- Spec -> [Spec]
- end,
- Result =
- case catch ct_testspec:collect_tests_from_file(TS1,[Node],AllowUserTerms) of
- {error,Reason} ->
- {error,Reason};
- TSRec=#testspec{logdir=AllLogDirs,
- config=StdCfgFiles,
- init=AllInitOpts,
- include=AllIncludes,
- userconfig=UserCfgFiles,
- event_handler=AllEvHs} ->
- AllCfgFiles = {StdCfgFiles, UserCfgFiles},
- {Run,Skip} = ct_testspec:prepare_tests(TSRec,Node),
- run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs,
- AllIncludes, [],[],AllInitOpts,TS1)
- end,
- [{TS,Result} | run_on_node(TestSpecs,AllowUserTerms,Node)];
+ case catch ct_testspec:collect_tests_from_file([TS],[Node],
+ AllowUserTerms) of
+ {error,Reason} ->
+ [{error,Reason} | run_on_node(TestSpecs,AllowUserTerms,Node)];
+ Tests ->
+ RunResult =
+ lists:map(
+ fun({Specs,TSRec=#testspec{logdir=AllLogDirs,
+ config=StdCfgFiles,
+ init=AllInitOpts,
+ include=AllIncludes,
+ userconfig=UserCfgFiles,
+ event_handler=AllEvHs}}) ->
+ AllCfgFiles = {StdCfgFiles,UserCfgFiles},
+ {Run,Skip} = ct_testspec:prepare_tests(TSRec,Node),
+ TSList = if is_integer(hd(TS)) -> [TS];
+ true -> TS end,
+ {Specs,run_all([{Node,Run,Skip}],AllLogDirs,
+ AllCfgFiles,AllEvHs,
+ AllIncludes, [],[],AllInitOpts,TSList)}
+ end, Tests),
+ RunResult ++ run_on_node(TestSpecs,AllowUserTerms,Node)
+ end;
run_on_node([],_,_) ->
[];
run_on_node(TS,AllowUserTerms,Node) when is_atom(Node) ->
@@ -245,7 +257,8 @@ run_all([],AllLogDirs,_,AllEvHs,_AllIncludes,
false -> "."
end,
log(tty,"Master Logdir","~ts",[MasterLogDir]),
- start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,LogDirs,InitOptions,Specs),
+ start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,
+ LogDirs,InitOptions,Specs),
ok.
@@ -297,13 +310,15 @@ start_master(NodeOptsList) ->
start_master(NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Specs) ->
Master = spawn_link(?MODULE,init_master,[self(),NodeOptsList,EvHandlers,
- MasterLogDir,LogDirs,InitOptions,Specs]),
+ MasterLogDir,LogDirs,
+ InitOptions,Specs]),
receive
{Master,Result} -> Result
end.
%%% @hidden
-init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Specs) ->
+init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,
+ InitOptions,Specs) ->
case whereis(ct_master) of
undefined ->
register(ct_master,self()),
@@ -325,6 +340,7 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Spec
{MLPid,_} = ct_master_logs:start(MasterLogDir,
[N || {N,_} <- NodeOptsList]),
log(all,"Master Logger process started","~w",[MLPid]),
+
case Specs of
[] -> ok;
_ ->
@@ -359,7 +375,8 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Spec
init_master1(Parent,NodeOptsList,InitOptions,LogDirs).
init_master1(Parent,NodeOptsList,InitOptions,LogDirs) ->
- {Inaccessible,NodeOptsList1,InitOptions1} = init_nodes(NodeOptsList,InitOptions),
+ {Inaccessible,NodeOptsList1,InitOptions1} = init_nodes(NodeOptsList,
+ InitOptions),
case Inaccessible of
[] ->
init_master2(Parent,NodeOptsList,LogDirs);
@@ -392,7 +409,8 @@ init_master2(Parent,NodeOptsList,LogDirs) ->
fun({Node,Opts}) ->
monitor_node(Node,true),
log(all,"Test Info","Starting test(s) on ~w...",[Node]),
- {spawn_link(Node,?MODULE,init_node_ctrl,[self(),Cookie,Opts]),Node}
+ {spawn_link(Node,?MODULE,init_node_ctrl,[self(),Cookie,Opts]),
+ Node}
end,
NodeCtrlPids = lists:map(SpawnAndMon,NodeOptsList),
Result = master_loop(#state{node_ctrl_pids=NodeCtrlPids,
@@ -438,11 +456,13 @@ master_loop(State=#state{node_ctrl_pids=NodeCtrlPids,
Bad
end,
log(all,"Test Info",
- "Test on node ~w failed! Reason: ~p",[Node,Error]),
+ "Test on node ~w failed! Reason: ~p",
+ [Node,Error]),
{Locks1,Blocked1} =
update_queue(exit,Node,Locks,Blocked),
master_loop(State#state{node_ctrl_pids=NodeCtrlPids1,
- results=[{Node,Error}|Results],
+ results=[{Node,
+ Error}|Results],
locks=Locks1,
blocked=Blocked1})
end;
@@ -754,7 +774,8 @@ start_nodes(InitOptions)->
IsAlive = lists:member(NodeName, nodes()),
case {HasNodeStart, IsAlive} of
{false, false}->
- io:format("WARNING: Node ~w is not alive but has no node_start option~n", [NodeName]);
+ io:format("WARNING: Node ~w is not alive but has no "
+ "node_start option~n", [NodeName]);
{false, true}->
io:format("Node ~w is alive~n", [NodeName]);
{true, false}->
@@ -763,12 +784,15 @@ start_nodes(InitOptions)->
lists:keytake(callback_module, 1, NodeStart),
case Callback:start(Host, Node, NodeStart2) of
{ok, NodeName} ->
- io:format("Node ~w started successfully with callback ~w~n", [NodeName,Callback]);
+ io:format("Node ~w started successfully "
+ "with callback ~w~n", [NodeName,Callback]);
{error, Reason, _NodeName} ->
- io:format("Failed to start node ~w with callback ~w! Reason: ~p~n", [NodeName, Callback, Reason])
+ io:format("Failed to start node ~w with callback ~w! "
+ "Reason: ~p~n", [NodeName, Callback, Reason])
end;
{true, true}->
- io:format("WARNING: Node ~w is alive but has node_start option~n", [NodeName])
+ io:format("WARNING: Node ~w is alive but has node_start "
+ "option~n", [NodeName])
end
end,
InitOptions).
@@ -781,7 +805,8 @@ eval_on_nodes(InitOptions)->
{false,_}->
ok;
{true,false}->
- io:format("WARNING: Node ~w is not alive but has eval option ~n", [NodeName]);
+ io:format("WARNING: Node ~w is not alive but has eval "
+ "option~n", [NodeName]);
{true,true}->
{eval, MFAs} = lists:keyfind(eval, 1, Options),
evaluate(NodeName, MFAs)
@@ -792,9 +817,11 @@ eval_on_nodes(InitOptions)->
evaluate(Node, [{M,F,A}|MFAs])->
case rpc:call(Node, M, F, A) of
{badrpc,Reason}->
- io:format("WARNING: Failed to call ~w:~w/~w on node ~w due to ~p~n", [M,F,length(A),Node,Reason]);
+ io:format("WARNING: Failed to call ~w:~w/~w on node ~w "
+ "due to ~p~n", [M,F,length(A),Node,Reason]);
Result->
- io:format("Called ~w:~w/~w on node ~w, result: ~p~n", [M,F,length(A),Node,Result])
+ io:format("Called ~w:~w/~w on node ~w, result: ~p~n",
+ [M,F,length(A),Node,Result])
end,
evaluate(Node, MFAs);
evaluate(_Node, [])->
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index acea6b5cb5..c0bdbb2a09 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -330,33 +330,33 @@ script_start1(Parent, Args) ->
true
end,
- StartOpts = #opts{label = Label, profile = Profile,
- vts = Vts, shell = Shell,
- cover = Cover, cover_stop = CoverStop,
- logdir = LogDir, logopts = LogOpts,
- basic_html = BasicHtml,
- verbosity = Verbosity,
- event_handlers = EvHandlers,
- ct_hooks = CTHooks,
- enable_builtin_hooks = EnableBuiltinHooks,
- auto_compile = AutoCompile,
- include = IncludeDirs,
- silent_connections = SilentConns,
- stylesheet = Stylesheet,
- multiply_timetraps = MultTT,
- scale_timetraps = ScaleTT,
- create_priv_dir = CreatePrivDir,
- starter = script},
-
+ Opts = #opts{label = Label, profile = Profile,
+ vts = Vts, shell = Shell,
+ cover = Cover, cover_stop = CoverStop,
+ logdir = LogDir, logopts = LogOpts,
+ basic_html = BasicHtml,
+ verbosity = Verbosity,
+ event_handlers = EvHandlers,
+ ct_hooks = CTHooks,
+ enable_builtin_hooks = EnableBuiltinHooks,
+ auto_compile = AutoCompile,
+ include = IncludeDirs,
+ silent_connections = SilentConns,
+ stylesheet = Stylesheet,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT,
+ create_priv_dir = CreatePrivDir,
+ starter = script},
+
%% check if log files should be refreshed or go on to run tests...
- Result = run_or_refresh(StartOpts, Args),
+ Result = run_or_refresh(Opts, Args),
%% send final results to starting process waiting in script_start/0
Parent ! {self(), Result}.
-run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
+run_or_refresh(Opts = #opts{logdir = LogDir}, Args) ->
case proplists:get_value(refresh_logs, Args) of
undefined ->
- script_start2(StartOpts, Args);
+ script_start2(Opts, Args);
Refresh ->
LogDir1 = case Refresh of
[] -> which(logdir,LogDir);
@@ -378,171 +378,203 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
{error,{all_suites_index,ASReason}};
_ ->
file:set_cwd(Cwd),
- io:format("Logs in ~ts refreshed!~n~n", [LogDir1]),
+ io:format("Logs in ~ts refreshed!~n~n",
+ [LogDir1]),
timer:sleep(500), % time to flush io before quitting
ok
end
end
end.
-script_start2(StartOpts = #opts{vts = undefined,
- shell = undefined}, Args) ->
- TestSpec = proplists:get_value(spec, Args),
- {Terms,Opts} =
- case TestSpec of
- Specs when Specs =/= [], Specs =/= undefined ->
- %% using testspec as input for test
- Relaxed = get_start_opt(allow_user_terms, true, false, Args),
- case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of
- {E,Reason} when E == error ; E == 'EXIT' ->
- {{error,Reason},StartOpts};
- TS ->
- SpecStartOpts = get_data_for_node(TS, node()),
-
- Label = choose_val(StartOpts#opts.label,
- SpecStartOpts#opts.label),
-
- Profile = choose_val(StartOpts#opts.profile,
- SpecStartOpts#opts.profile),
-
- LogDir = choose_val(StartOpts#opts.logdir,
- SpecStartOpts#opts.logdir),
-
- AllLogOpts = merge_vals([StartOpts#opts.logopts,
- SpecStartOpts#opts.logopts]),
- AllVerbosity =
- merge_keyvals([StartOpts#opts.verbosity,
- SpecStartOpts#opts.verbosity]),
- AllSilentConns =
- merge_vals([StartOpts#opts.silent_connections,
- SpecStartOpts#opts.silent_connections]),
- Cover =
- choose_val(StartOpts#opts.cover,
- SpecStartOpts#opts.cover),
- CoverStop =
- choose_val(StartOpts#opts.cover_stop,
- SpecStartOpts#opts.cover_stop),
- MultTT =
- choose_val(StartOpts#opts.multiply_timetraps,
- SpecStartOpts#opts.multiply_timetraps),
- ScaleTT =
- choose_val(StartOpts#opts.scale_timetraps,
- SpecStartOpts#opts.scale_timetraps),
-
- CreatePrivDir =
- choose_val(StartOpts#opts.create_priv_dir,
- SpecStartOpts#opts.create_priv_dir),
-
- AllEvHs =
- merge_vals([StartOpts#opts.event_handlers,
- SpecStartOpts#opts.event_handlers]),
-
- AllCTHooks = merge_vals(
- [StartOpts#opts.ct_hooks,
- SpecStartOpts#opts.ct_hooks]),
-
- EnableBuiltinHooks =
- choose_val(
- StartOpts#opts.enable_builtin_hooks,
- SpecStartOpts#opts.enable_builtin_hooks),
-
- Stylesheet =
- choose_val(StartOpts#opts.stylesheet,
- SpecStartOpts#opts.stylesheet),
-
- AllInclude = merge_vals([StartOpts#opts.include,
- SpecStartOpts#opts.include]),
- application:set_env(common_test, include, AllInclude),
-
- AutoCompile =
- case choose_val(StartOpts#opts.auto_compile,
- SpecStartOpts#opts.auto_compile) of
- undefined ->
- true;
- ACBool ->
- application:set_env(common_test,
- auto_compile,
- ACBool),
- ACBool
- end,
-
- BasicHtml =
- case choose_val(StartOpts#opts.basic_html,
- SpecStartOpts#opts.basic_html) of
- undefined ->
- false;
- BHBool ->
- application:set_env(common_test, basic_html,
- BHBool),
- BHBool
- end,
-
- {TS,StartOpts#opts{label = Label,
- profile = Profile,
- testspecs = Specs,
- cover = Cover,
- cover_stop = CoverStop,
- logdir = LogDir,
- logopts = AllLogOpts,
- basic_html = BasicHtml,
- verbosity = AllVerbosity,
- silent_connections = AllSilentConns,
- config = SpecStartOpts#opts.config,
- event_handlers = AllEvHs,
- ct_hooks = AllCTHooks,
- enable_builtin_hooks =
- EnableBuiltinHooks,
- stylesheet = Stylesheet,
- auto_compile = AutoCompile,
- include = AllInclude,
- multiply_timetraps = MultTT,
- scale_timetraps = ScaleTT,
- create_priv_dir = CreatePrivDir}}
- end;
- _ ->
- {undefined,StartOpts}
- end,
- %% read config/userconfig from start flags
- InitConfig = ct_config:prepare_config_list(Args),
- TheLogDir = which(logdir, Opts#opts.logdir),
- case {TestSpec,Terms} of
- {_,{error,_}=Error} ->
- Error;
- {[],_} ->
+script_start2(Opts = #opts{vts = undefined,
+ shell = undefined}, Args) ->
+ case proplists:get_value(spec, Args) of
+ Specs when Specs =/= [], Specs =/= undefined ->
+ Specs1 = get_start_opt(join_specs, [Specs], Specs, Args),
+ %% using testspec as input for test
+ Relaxed = get_start_opt(allow_user_terms, true, false, Args),
+ case catch ct_testspec:collect_tests_from_file(Specs1, Relaxed) of
+ {E,Reason} when E == error ; E == 'EXIT' ->
+ {error,Reason};
+ TestSpecData ->
+ execute_all_specs(TestSpecData, Opts, Args, [])
+ end;
+ [] ->
{error,no_testspec_specified};
- {undefined,_} -> % no testspec used
- case check_and_install_configfiles(InitConfig, TheLogDir, Opts) of
+ _ -> % no testspec used
+ %% read config/userconfig from start flags
+ InitConfig = ct_config:prepare_config_list(Args),
+ TheLogDir = which(logdir, Opts#opts.logdir),
+ case check_and_install_configfiles(InitConfig,
+ TheLogDir,
+ Opts) of
ok -> % go on read tests from start flags
script_start3(Opts#opts{config=InitConfig,
logdir=TheLogDir}, Args);
Error ->
Error
- end;
- {_,_} -> % testspec used
- %% merge config from start flags with config from testspec
- AllConfig = merge_vals([InitConfig, Opts#opts.config]),
- case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
- ok -> % read tests from spec
- {Run,Skip} = ct_testspec:prepare_tests(Terms, node()),
- do_run(Run, Skip, Opts#opts{config=AllConfig,
- logdir=TheLogDir}, Args);
- Error ->
- Error
end
end;
-script_start2(StartOpts, Args) ->
+script_start2(Opts, Args) ->
%% read config/userconfig from start flags
InitConfig = ct_config:prepare_config_list(Args),
- LogDir = which(logdir, StartOpts#opts.logdir),
- case check_and_install_configfiles(InitConfig, LogDir, StartOpts) of
+ LogDir = which(logdir, Opts#opts.logdir),
+ case check_and_install_configfiles(InitConfig, LogDir, Opts) of
ok -> % go on read tests from start flags
- script_start3(StartOpts#opts{config=InitConfig,
- logdir=LogDir}, Args);
+ script_start3(Opts#opts{config=InitConfig,
+ logdir=LogDir}, Args);
+ Error ->
+ Error
+ end.
+
+execute_all_specs([], _, _, Result) ->
+ Result1 = lists:reverse(Result),
+ case lists:keysearch('EXIT', 1, Result1) of
+ {value,{_,_,ExitReason}} ->
+ exit(ExitReason);
+ false ->
+ case lists:keysearch(error, 1, Result1) of
+ {value,Error} ->
+ Error;
+ false ->
+ lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}},
+ {Ok1,Fail1,{UserSkip1,AutoSkip1}}) ->
+ {Ok1+Ok,Fail1+Fail,
+ {UserSkip1+UserSkip,
+ AutoSkip1+AutoSkip}}
+ end, {0,0,{0,0}}, Result1)
+ end
+ end;
+
+execute_all_specs([{Specs,TS} | TSs], Opts, Args, Result) ->
+ CombinedOpts = combine_test_opts(TS, Specs, Opts),
+ try execute_one_spec(TS, CombinedOpts, Args) of
+ ExecResult ->
+ execute_all_specs(TSs, Opts, Args, [ExecResult|Result])
+ catch
+ _ : ExitReason ->
+ execute_all_specs(TSs, Opts, Args,
+ [{'EXIT',self(),ExitReason}|Result])
+ end.
+
+execute_one_spec(TS, Opts, Args) ->
+ %% read config/userconfig from start flags
+ InitConfig = ct_config:prepare_config_list(Args),
+ TheLogDir = which(logdir, Opts#opts.logdir),
+ %% merge config from start flags with config from testspec
+ AllConfig = merge_vals([InitConfig, Opts#opts.config]),
+ case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
+ ok -> % read tests from spec
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ do_run(Run, Skip, Opts#opts{config=AllConfig,
+ logdir=TheLogDir}, Args);
Error ->
Error
end.
+combine_test_opts(TS, Specs, Opts) ->
+ TSOpts = get_data_for_node(TS, node()),
+
+ Label = choose_val(Opts#opts.label,
+ TSOpts#opts.label),
+
+ Profile = choose_val(Opts#opts.profile,
+ TSOpts#opts.profile),
+
+ LogDir = choose_val(Opts#opts.logdir,
+ TSOpts#opts.logdir),
+
+ AllLogOpts = merge_vals([Opts#opts.logopts,
+ TSOpts#opts.logopts]),
+ AllVerbosity =
+ merge_keyvals([Opts#opts.verbosity,
+ TSOpts#opts.verbosity]),
+ AllSilentConns =
+ merge_vals([Opts#opts.silent_connections,
+ TSOpts#opts.silent_connections]),
+ Cover =
+ choose_val(Opts#opts.cover,
+ TSOpts#opts.cover),
+ CoverStop =
+ choose_val(Opts#opts.cover_stop,
+ TSOpts#opts.cover_stop),
+ MultTT =
+ choose_val(Opts#opts.multiply_timetraps,
+ TSOpts#opts.multiply_timetraps),
+ ScaleTT =
+ choose_val(Opts#opts.scale_timetraps,
+ TSOpts#opts.scale_timetraps),
+
+ CreatePrivDir =
+ choose_val(Opts#opts.create_priv_dir,
+ TSOpts#opts.create_priv_dir),
+
+ AllEvHs =
+ merge_vals([Opts#opts.event_handlers,
+ TSOpts#opts.event_handlers]),
+
+ AllCTHooks = merge_vals(
+ [Opts#opts.ct_hooks,
+ TSOpts#opts.ct_hooks]),
+
+ EnableBuiltinHooks =
+ choose_val(
+ Opts#opts.enable_builtin_hooks,
+ TSOpts#opts.enable_builtin_hooks),
+
+ Stylesheet =
+ choose_val(Opts#opts.stylesheet,
+ TSOpts#opts.stylesheet),
+
+ AllInclude = merge_vals([Opts#opts.include,
+ TSOpts#opts.include]),
+ application:set_env(common_test, include, AllInclude),
+
+ AutoCompile =
+ case choose_val(Opts#opts.auto_compile,
+ TSOpts#opts.auto_compile) of
+ undefined ->
+ true;
+ ACBool ->
+ application:set_env(common_test,
+ auto_compile,
+ ACBool),
+ ACBool
+ end,
+
+ BasicHtml =
+ case choose_val(Opts#opts.basic_html,
+ TSOpts#opts.basic_html) of
+ undefined ->
+ false;
+ BHBool ->
+ application:set_env(common_test, basic_html,
+ BHBool),
+ BHBool
+ end,
+
+ Opts#opts{label = Label,
+ profile = Profile,
+ testspecs = Specs,
+ cover = Cover,
+ cover_stop = CoverStop,
+ logdir = which(logdir, LogDir),
+ logopts = AllLogOpts,
+ basic_html = BasicHtml,
+ verbosity = AllVerbosity,
+ silent_connections = AllSilentConns,
+ config = TSOpts#opts.config,
+ event_handlers = AllEvHs,
+ ct_hooks = AllCTHooks,
+ enable_builtin_hooks = EnableBuiltinHooks,
+ stylesheet = Stylesheet,
+ auto_compile = AutoCompile,
+ include = AllInclude,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT,
+ create_priv_dir = CreatePrivDir}.
+
check_and_install_configfiles(
Configs, LogDir, #opts{
event_handlers = EvHandlers,
@@ -562,12 +594,12 @@ check_and_install_configfiles(
{error,{cant_load_callback_module,Info}}
end.
-script_start3(StartOpts, Args) ->
- StartOpts1 = get_start_opt(step,
- fun(Step) ->
- StartOpts#opts{step = Step,
- cover = undefined}
- end, StartOpts, Args),
+script_start3(Opts, Args) ->
+ Opts1 = get_start_opt(step,
+ fun(Step) ->
+ Opts#opts{step = Step,
+ cover = undefined}
+ end, Opts, Args),
case {proplists:get_value(dir, Args),
proplists:get_value(suite, Args),
groups_and_cases(proplists:get_value(group, Args),
@@ -581,17 +613,17 @@ script_start3(StartOpts, Args) ->
{error,no_dir_specified};
{Dirs,undefined,[]} when is_list(Dirs) ->
- script_start4(StartOpts#opts{tests = tests(Dirs)}, Args);
+ script_start4(Opts#opts{tests = tests(Dirs)}, Args);
{undefined,Suites,[]} when is_list(Suites) ->
Ts = tests([suite_to_test(S) || S <- Suites]),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
{undefined,Suite,GsAndCs} when is_list(Suite) ->
case [suite_to_test(S) || S <- Suite] of
DirMods = [_] ->
Ts = tests(DirMods, GsAndCs),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
[_,_|_] ->
{error,multiple_suites_and_cases};
_ ->
@@ -605,10 +637,10 @@ script_start3(StartOpts, Args) ->
case [suite_to_test(Dir,S) || S <- Suite] of
DirMods when GsAndCs == [] ->
Ts = tests(DirMods),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
DirMods = [_] when GsAndCs /= [] ->
Ts = tests(DirMods, GsAndCs),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
[_,_|_] when GsAndCs /= [] ->
{error,multiple_suites_and_cases};
_ ->
@@ -619,8 +651,8 @@ script_start3(StartOpts, Args) ->
{error,incorrect_start_options};
{undefined,undefined,_} ->
- if StartOpts#opts.vts ; StartOpts#opts.shell ->
- script_start4(StartOpts#opts{tests = []}, Args);
+ if Opts#opts.vts ; Opts#opts.shell ->
+ script_start4(Opts#opts{tests = []}, Args);
true ->
script_usage(),
{error,missing_start_options}
@@ -750,6 +782,7 @@ script_usage() ->
"\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
"\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]"
"\n\t[-allow_user_terms]"
+ "\n\t[-join_specs]"
"\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
@@ -1044,105 +1077,60 @@ run_test2(StartOpts) ->
end.
run_spec_file(Relaxed,
- Opts = #opts{testspecs = Specs, config = CfgFiles},
+ Opts = #opts{testspecs = Specs},
StartOpts) ->
Specs1 = case Specs of
[X|_] when is_integer(X) -> [Specs];
_ -> Specs
end,
AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1),
- log_ts_names(AbsSpecs),
- case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of
+ AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts),
+ case catch ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of
{Error,CTReason} when Error == error ; Error == 'EXIT' ->
exit({error,CTReason});
- TS ->
- SpecOpts = get_data_for_node(TS, node()),
- Label = choose_val(Opts#opts.label,
- SpecOpts#opts.label),
- Profile = choose_val(Opts#opts.profile,
- SpecOpts#opts.profile),
- LogDir = choose_val(Opts#opts.logdir,
- SpecOpts#opts.logdir),
- AllLogOpts = merge_vals([Opts#opts.logopts,
- SpecOpts#opts.logopts]),
- Stylesheet = choose_val(Opts#opts.stylesheet,
- SpecOpts#opts.stylesheet),
- AllVerbosity = merge_keyvals([Opts#opts.verbosity,
- SpecOpts#opts.verbosity]),
- AllSilentConns = merge_vals([Opts#opts.silent_connections,
- SpecOpts#opts.silent_connections]),
- AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]),
- Cover = choose_val(Opts#opts.cover,
- SpecOpts#opts.cover),
- CoverStop = choose_val(Opts#opts.cover_stop,
- SpecOpts#opts.cover_stop),
- MultTT = choose_val(Opts#opts.multiply_timetraps,
- SpecOpts#opts.multiply_timetraps),
- ScaleTT = choose_val(Opts#opts.scale_timetraps,
- SpecOpts#opts.scale_timetraps),
- CreatePrivDir = choose_val(Opts#opts.create_priv_dir,
- SpecOpts#opts.create_priv_dir),
- AllEvHs = merge_vals([Opts#opts.event_handlers,
- SpecOpts#opts.event_handlers]),
- AllInclude = merge_vals([Opts#opts.include,
- SpecOpts#opts.include]),
- AllCTHooks = merge_vals([Opts#opts.ct_hooks,
- SpecOpts#opts.ct_hooks]),
- EnableBuiltinHooks = choose_val(Opts#opts.enable_builtin_hooks,
- SpecOpts#opts.enable_builtin_hooks),
-
- application:set_env(common_test, include, AllInclude),
-
- AutoCompile = case choose_val(Opts#opts.auto_compile,
- SpecOpts#opts.auto_compile) of
- undefined ->
- true;
- ACBool ->
- application:set_env(common_test, auto_compile,
- ACBool),
- ACBool
- end,
+ TestSpecData ->
+ run_all_specs(TestSpecData, Opts, StartOpts, [])
+ end.
- BasicHtml = case choose_val(Opts#opts.basic_html,
- SpecOpts#opts.basic_html) of
- undefined ->
- false;
- BHBool ->
- application:set_env(common_test, basic_html,
- BHBool),
- BHBool
- end,
-
- Opts1 = Opts#opts{label = Label,
- profile = Profile,
- cover = Cover,
- cover_stop = CoverStop,
- logdir = which(logdir, LogDir),
- logopts = AllLogOpts,
- stylesheet = Stylesheet,
- basic_html = BasicHtml,
- verbosity = AllVerbosity,
- silent_connections = AllSilentConns,
- config = AllConfig,
- event_handlers = AllEvHs,
- auto_compile = AutoCompile,
- include = AllInclude,
- testspecs = AbsSpecs,
- multiply_timetraps = MultTT,
- scale_timetraps = ScaleTT,
- create_priv_dir = CreatePrivDir,
- ct_hooks = AllCTHooks,
- enable_builtin_hooks = EnableBuiltinHooks
- },
-
- case check_and_install_configfiles(AllConfig,Opts1#opts.logdir,
- Opts1) of
- ok ->
- {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- reformat_result(catch do_run(Run, Skip, Opts1, StartOpts));
- {error,GCFReason} ->
- exit({error,GCFReason})
+run_all_specs([], _, _, TotResult) ->
+ TotResult1 = lists:reverse(TotResult),
+ case lists:keysearch('EXIT', 1, TotResult1) of
+ {value,{_,_,ExitReason}} ->
+ exit(ExitReason);
+ false ->
+ case lists:keysearch(error, 1, TotResult1) of
+ {value,Error} ->
+ Error;
+ false ->
+ lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}},
+ {Ok1,Fail1,{UserSkip1,AutoSkip1}}) ->
+ {Ok1+Ok,Fail1+Fail,
+ {UserSkip1+UserSkip,
+ AutoSkip1+AutoSkip}}
+ end, {0,0,{0,0}}, TotResult1)
end
+ end;
+
+run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) ->
+ log_ts_names(Specs),
+ Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts),
+ AllConfig = merge_vals([Opts#opts.config, TSConfig]),
+ try run_one_spec(TS, Combined#opts{config = AllConfig}, StartOpts) of
+ Result ->
+ run_all_specs(TSs, Opts, StartOpts, [Result | TotResult])
+ catch
+ _ : Reason ->
+ run_all_specs(TSs, Opts, StartOpts, [{error,Reason} | TotResult])
+ end.
+
+run_one_spec(TS, CombinedOpts, StartOpts) ->
+ #opts{logdir = Logdir, config = Config} = CombinedOpts,
+ case check_and_install_configfiles(Config, Logdir, CombinedOpts) of
+ ok ->
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ reformat_result(catch do_run(Run, Skip, CombinedOpts, StartOpts));
+ Error ->
+ Error
end.
run_prepared(Run, Skip, Opts = #opts{logdir = LogDir,
@@ -2896,6 +2884,10 @@ opts2args(EnvStartOpts) ->
[{allow_user_terms,[]}];
({allow_user_terms,false}) ->
[];
+ ({join_specs,true}) ->
+ [{join_specs,[]}];
+ ({join_specs,false}) ->
+ [];
({auto_compile,false}) ->
[{no_auto_compile,[]}];
({auto_compile,true}) ->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index af81edf90b..9b88168583 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -93,7 +93,7 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) ->
%% run_per_node/2 takes the Run list as input and returns a list
%% of {Node,RunPerNode,[]} tuples where the tests have been sorted
%% on a per node basis.
-run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) ->
+run_per_node([{{Node,Dir},Test}|Ts],Result,MergeTests) ->
{value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result),
Run1 = case MergeTests of
false ->
@@ -190,7 +190,7 @@ prepare_suites(_Node,_Dir,[],Run,Skip) ->
prepare_cases(Node,Dir,Suite,Cases) ->
case get_skipped_cases(Node,Dir,Suite,Cases) of
- SkipAll=[{{Node,Dir},{Suite,_Cmt}}] -> % all cases to be skipped
+ SkipAll=[{{Node,Dir},{Suite,_Cmt}}] -> % all cases to be skipped
%% note: this adds an 'all' test even if only skip is specified
{[{{Node,Dir},{Suite,all}}],SkipAll};
Skipped ->
@@ -241,34 +241,139 @@ get_skipped_cases1(_,_,_,[]) ->
%%% collect_tests_from_file reads a testspec file and returns a record
%%% containing the data found.
-collect_tests_from_file(Specs, Relaxed) ->
+collect_tests_from_file(Specs,Relaxed) ->
collect_tests_from_file(Specs,[node()],Relaxed).
collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) ->
NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes),
- catch collect_tests_from_file1(Specs,#testspec{nodes=NodeRefs},Relaxed).
+ %% [Spec1,Spec2,...] means create one testpec record per Spec file
+ %% [[Spec1,Spec2,...]] means merge all specs into one testspec record
+ {Join,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)};
+ true -> {false,Specs}
+ end,
+ Specs2 = [filename:absname(S) || S <- Specs1],
+ TS0 = #testspec{nodes=NodeRefs},
+
+ try create_specs(Specs2,TS0,Relaxed,Join) of
+ {{[],_},SeparateTestSpecs} ->
+ filter_and_convert(SeparateTestSpecs);
+ {{_,#testspec{tests=[]}},SeparateTestSpecs} ->
+ filter_and_convert(SeparateTestSpecs);
+ {Joined,SeparateTestSpecs} ->
+ [filter_and_convert(Joined) |
+ filter_and_convert(SeparateTestSpecs)]
+ catch
+ _:Error ->
+ Error
+ end.
+
+filter_and_convert(Joined) when is_tuple(Joined) ->
+ hd(filter_and_convert([Joined]));
+filter_and_convert([{_,#testspec{tests=[]}}|TSs]) ->
+ filter_and_convert(TSs);
+filter_and_convert([{[{SpecFile,MergeTests}|SMs],TestSpec}|TSs]) ->
+ #testspec{config = CfgFiles} = TestSpec,
+ TestSpec1 = TestSpec#testspec{config = delete_dups(CfgFiles),
+ merge_tests = MergeTests},
+ %% set the merge_tests value for the testspec to the value
+ %% of the first test spec in the set
+ [{[SpecFile | [SF || {SF,_} <- SMs]], TestSpec1} | filter_and_convert(TSs)];
+filter_and_convert([]) ->
+ [].
+
+delete_dups(Elems) ->
+ delete_dups1(lists:reverse(Elems),[]).
-collect_tests_from_file1([Spec|Specs],TestSpec,Relaxed) ->
+delete_dups1([E|Es],Keep) ->
+ case lists:member(E,Es) of
+ true ->
+ delete_dups1(Es,Keep);
+ false ->
+ delete_dups1(Es,[E|Keep])
+ end;
+delete_dups1([],Keep) ->
+ Keep.
+
+create_specs(Specs,TestSpec,Relaxed,Join) ->
+ SpecsTree = create_spec_tree(Specs,TestSpec,Join,[]),
+ create_specs(SpecsTree,TestSpec,Relaxed).
+
+create_spec_tree([Spec|Specs],TS,JoinWithNext,Known) ->
SpecDir = filename:dirname(filename:absname(Spec)),
- case file:consult(Spec) of
- {ok,Terms} ->
- case collect_tests(Terms,
- TestSpec#testspec{spec_dir=SpecDir},
- Relaxed) of
- TS = #testspec{tests=Tests, logdir=LogDirs} when Specs == [] ->
- LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
- TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1};
- TS = #testspec{alias = As, nodes = Ns} ->
- TS1 = TS#testspec{alias = lists:reverse(As),
- nodes = lists:reverse(Ns)},
- collect_tests_from_file1(Specs,TS1,Relaxed)
- end;
- {error,Reason} ->
- ReasonStr =
- lists:flatten(io_lib:format("~ts",
- [file:format_error(Reason)])),
- throw({error,{Spec,ReasonStr}})
- end.
+ TS1 = TS#testspec{spec_dir=SpecDir},
+ SpecAbsName = get_absfile(Spec,TS1),
+ case lists:member(SpecAbsName,Known) of
+ true ->
+ throw({error,{cyclic_reference,SpecAbsName}});
+ false ->
+ case file:consult(SpecAbsName) of
+ {ok,Terms} ->
+ Terms1 = replace_names(Terms),
+ {InclJoin,InclSep} = get_included_specs(Terms1,TS1),
+ {SpecAbsName,Terms1,
+ create_spec_tree(InclJoin,TS,true,[SpecAbsName|Known]),
+ create_spec_tree(InclSep,TS,false,[SpecAbsName|Known]),
+ JoinWithNext,
+ create_spec_tree(Specs,TS,JoinWithNext,Known)};
+ {error,Reason} ->
+ ReasonStr =
+ lists:flatten(io_lib:format("~s",
+ [file:format_error(Reason)])),
+ throw({error,{SpecAbsName,ReasonStr}})
+ end
+ end;
+create_spec_tree([],_TS,_JoinWithNext,_Known) ->
+ [].
+
+create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec},
+ TestSpec,Relaxed) ->
+ SpecDir = filename:dirname(filename:absname(Spec)),
+ TestSpec1 = create_spec(Terms,TestSpec#testspec{spec_dir=SpecDir},Relaxed),
+
+ {{JoinSpecs1,JoinTS1},Separate1} = create_specs(InclJoin,TestSpec1,Relaxed),
+ {{JoinSpecs2,JoinTS2},Separate2} =
+ case JoinWithNext of
+ true ->
+ create_specs(NextSpec,JoinTS1,Relaxed);
+ false ->
+ {{[],JoinTS1},[]}
+ end,
+ {SepJoinSpecs,Separate3} = create_specs(InclSep,TestSpec,Relaxed),
+ {SepJoinSpecs1,Separate4} =
+ case JoinWithNext of
+ true ->
+ {{[],TestSpec},[]};
+ false ->
+ create_specs(NextSpec,TestSpec,Relaxed)
+ end,
+
+ SpecInfo = {Spec,TestSpec1#testspec.merge_tests},
+ AllSeparate =
+ [TSData || TSData = {Ss,_TS} <- Separate3++Separate1++
+ [SepJoinSpecs]++Separate2++
+ Separate4++[SepJoinSpecs1],
+ Ss /= []],
+
+ case {JoinWithNext,JoinSpecs1} of
+ {true,_} ->
+ {{[SpecInfo|(JoinSpecs1++JoinSpecs2)],JoinTS2},
+ AllSeparate};
+ {false,[]} ->
+ {{[],TestSpec},
+ [{[SpecInfo],TestSpec1}|AllSeparate]};
+ {false,_} ->
+ {{[SpecInfo|(JoinSpecs1++JoinSpecs2)],JoinTS2},
+ AllSeparate}
+ end;
+create_specs([],TestSpec,_Relaxed) ->
+ {{[],TestSpec},[]}.
+
+create_spec(Terms,TestSpec,Relaxed) ->
+ TS = #testspec{tests=Tests, logdir=LogDirs} =
+ collect_tests({false,Terms},TestSpec,Relaxed),
+ LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
+ TS#testspec{tests=lists:flatten(Tests),
+ logdir=LogDirs1}.
collect_tests_from_list(Terms,Relaxed) ->
collect_tests_from_list(Terms,[node()],Relaxed).
@@ -276,8 +381,8 @@ collect_tests_from_list(Terms,Relaxed) ->
collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) ->
{ok,Cwd} = file:get_cwd(),
NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes),
- case catch collect_tests(Terms,#testspec{nodes=NodeRefs,
- spec_dir=Cwd},
+ case catch collect_tests({true,Terms},#testspec{nodes=NodeRefs,
+ spec_dir=Cwd},
Relaxed) of
E = {error,_} ->
E;
@@ -287,10 +392,16 @@ collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) ->
TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1}
end.
-collect_tests(Terms,TestSpec,Relaxed) ->
+collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) ->
put(relaxed,Relaxed),
- Terms1 = replace_names(Terms),
- TestSpec1 = get_global(Terms1,TestSpec),
+ Terms1 = if Replace -> replace_names(Terms);
+ true -> Terms
+ end,
+ %% reverse nodes and aliases initially to get the order of them right
+ %% in case this spec is being joined with a previous one
+ TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As),
+ nodes = lists:reverse(Ns),
+ merge_tests = true}),
TestSpec2 = get_all_nodes(Terms1,TestSpec1),
{Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2),
add_tests(Terms2,TestSpec3).
@@ -420,9 +531,30 @@ replace_names_in_node1(NodeStr,Defs=[{Name,Replacement}|Ds]) ->
replace_names_in_node1(NodeStr,[]) ->
NodeStr.
+%% look for other specification files, either to join with the
+%% current spec, or execute as separate test runs
+get_included_specs(Terms,TestSpec) ->
+ get_included_specs(Terms,TestSpec,[],[]).
+
+get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Sep) ->
+ Specs = case SpecOrSpecs of
+ [File|_] when is_list(File) ->
+ [get_absfile(Spec,TestSpec) || Spec <- SpecOrSpecs];
+ [Ch|_] when is_integer(Ch) ->
+ [get_absfile(SpecOrSpecs,TestSpec)]
+ end,
+ if How == join ->
+ get_included_specs(Ts,TestSpec,Join++Specs,Sep);
+ true ->
+ get_included_specs(Ts,TestSpec,Join,Sep++Specs)
+ end;
+get_included_specs([_|Ts],TestSpec,Join,Sep) ->
+ get_included_specs(Ts,TestSpec,Join,Sep);
+get_included_specs([],_,Join,Sep) ->
+ {Join,Sep}.
%% global terms that will be used for analysing all other terms in the spec
-get_global([{merge_tests,Bool} | Ts], Spec) ->
+get_global([{merge_tests,Bool}|Ts],Spec) ->
get_global(Ts,Spec#testspec{merge_tests=Bool});
%% the 'define' term replaces the 'alias' and 'node' terms, but we need to keep
@@ -637,7 +769,7 @@ add_tests([{suites,all_nodes,Dir,Ss}|Ts],Spec) ->
add_tests([{suites,Dir,Ss}|Ts],Spec) ->
add_tests([{suites,all_nodes,Dir,Ss}|Ts],Spec);
add_tests([{suites,Nodes,Dir,Ss}|Ts],Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,suites,[Dir,Ss],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(Nodes,suites,[Dir,Ss],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{suites,Node,Dir,Ss}|Ts],Spec) ->
Tests = Spec#testspec.tests,
@@ -660,11 +792,11 @@ add_tests([{groups,Dir,Suite,Gs}|Ts],Spec) ->
add_tests([{groups,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) ->
add_tests([{groups,all_nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec);
add_tests([{groups,Nodes,Dir,Suite,Gs}|Ts],Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,groups,[Dir,Suite,Gs],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(Nodes,groups,[Dir,Suite,Gs],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{groups,Nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],
Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,groups,[Dir,Suite,Gs,{cases,TCs}],Ts,
+ Ts1 = per_node(Nodes,groups,[Dir,Suite,Gs,{cases,TCs}],Ts,
Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) ->
@@ -688,7 +820,7 @@ add_tests([{cases,all_nodes,Dir,Suite,Cs}|Ts],Spec) ->
add_tests([{cases,Dir,Suite,Cs}|Ts],Spec) ->
add_tests([{cases,all_nodes,Dir,Suite,Cs}|Ts],Spec);
add_tests([{cases,Nodes,Dir,Suite,Cs}|Ts],Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,cases,[Dir,Suite,Cs],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(Nodes,cases,[Dir,Suite,Cs],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) ->
Tests = Spec#testspec.tests,
@@ -703,7 +835,7 @@ add_tests([{skip_suites,all_nodes,Dir,Ss,Cmt}|Ts],Spec) ->
add_tests([{skip_suites,Dir,Ss,Cmt}|Ts],Spec) ->
add_tests([{skip_suites,all_nodes,Dir,Ss,Cmt}|Ts],Spec);
add_tests([{skip_suites,Nodes,Dir,Ss,Cmt}|Ts],Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,skip_suites,[Dir,Ss,Cmt],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(Nodes,skip_suites,[Dir,Ss,Cmt],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
@@ -724,11 +856,11 @@ add_tests([{skip_groups,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
add_tests([{skip_groups,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) ->
add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec);
add_tests([{skip_groups,Nodes,Dir,Suite,Gs,Cmt}|Ts],Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,Cmt],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(Nodes,skip_groups,[Dir,Suite,Gs,Cmt],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{skip_groups,Nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],
Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,{cases,TCs},Cmt],Ts,
+ Ts1 = per_node(Nodes,skip_groups,[Dir,Suite,Gs,{cases,TCs},Cmt],Ts,
Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
@@ -752,7 +884,7 @@ add_tests([{skip_cases,all_nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
add_tests([{skip_cases,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
add_tests([{skip_cases,all_nodes,Dir,Suite,Cs,Cmt}|Ts],Spec);
add_tests([{skip_cases,Nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) when is_list(Nodes) ->
- Ts1 = separate(Nodes,skip_cases,[Dir,Suite,Cs,Cmt],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(Nodes,skip_cases,[Dir,Suite,Cs,Cmt],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
@@ -792,7 +924,10 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled
add_tests([{node,_,_}|Ts],Spec) -> % handled
add_tests(Ts,Spec);
-add_tests([{merge_tests, _} | Ts], Spec) -> % handled
+add_tests([{merge_tests,_} | Ts], Spec) -> % handled
+ add_tests(Ts,Spec);
+
+add_tests([{specs,_,_} | Ts], Spec) -> % handled
add_tests(Ts,Spec);
%% --------------------------------------------------
@@ -821,7 +956,7 @@ add_tests([{Tag,NodesOrOther,Data}|Ts],Spec) when is_list(NodesOrOther) ->
case lists:all(fun(Test) -> is_node(Test,Spec#testspec.nodes)
end, NodesOrOther) of
true ->
- Ts1 = separate(NodesOrOther,Tag,[Data],Ts,Spec#testspec.nodes),
+ Ts1 = per_node(NodesOrOther,Tag,[Data],Ts,Spec#testspec.nodes),
add_tests(Ts1,Spec);
false ->
add_tests([{Tag,all_nodes,{NodesOrOther,Data}}|Ts],Spec)
@@ -977,12 +1112,12 @@ update_recorded(Tag,Node,Spec) ->
end.
%% create one test term per node
-separate(Nodes,Tag,Data,Tests,Refs) ->
- Separated = separate(Nodes,Tag,Data,Refs),
+per_node(Nodes,Tag,Data,Tests,Refs) ->
+ Separated = per_node(Nodes,Tag,Data,Refs),
Separated ++ Tests.
-separate([N|Ns],Tag,Data,Refs) ->
- [list_to_tuple([Tag,ref2node(N,Refs)|Data])|separate(Ns,Tag,Data,Refs)];
-separate([],_,_,_) ->
+per_node([N|Ns],Tag,Data,Refs) ->
+ [list_to_tuple([Tag,ref2node(N,Refs)|Data])|per_node(Ns,Tag,Data,Refs)];
+per_node([],_,_,_) ->
[].
%% read the value for FieldName in record Rec#testspec
@@ -1039,14 +1174,21 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when
{[Gr],Cases};
true ->
{Gr,Cases} end || Gr <- Groups],
- case lists:keysearch({Node,Dir},1,Tests) of
- {value,{{Node,Dir},[{all,_}]}} ->
- Tests;
- {value,{{Node,Dir},Suites0}} ->
- Suites1 = insert_groups1(Suite,Groups1,Suites0),
- insert_in_order({{Node,Dir},Suites1},Tests);
- false ->
- insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests)
+ {Tests1,Done} =
+ lists:foldr(fun(All={{N,D},[{all,_}]},{Replaced,_}) when N == Node,
+ D == Dir ->
+ {[All|Replaced],true};
+ ({{N,D},Suites0},{Replaced,_}) when N == Node,
+ D == Dir ->
+ Suites1 = insert_groups1(Suite,Groups1,Suites0),
+ {[{{N,D},Suites1}|Replaced],true};
+ (T,{Replaced,Match}) ->
+ {[T|Replaced],Match}
+ end, {[],false}, Tests),
+ if not Done ->
+ Tests ++ [{{Node,Dir},[{Suite,Groups1}]}];
+ true ->
+ Tests1
end;
insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests)
when is_atom(Case) ->
@@ -1084,14 +1226,21 @@ insert_groups2([],GrAndCases) ->
insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) ->
append({{Node,Dir},[{Suite,Cases}]},Tests);
insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) ->
- case lists:keysearch({Node,Dir},1,Tests) of
- {value,{{Node,Dir},[{all,_}]}} ->
- Tests;
- {value,{{Node,Dir},Suites0}} ->
- Suites1 = insert_cases1(Suite,Cases,Suites0),
- insert_in_order({{Node,Dir},Suites1},Tests);
- false ->
- insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests)
+ {Tests1,Done} =
+ lists:foldr(fun(All={{N,D},[{all,_}]},{Replaced,_}) when N == Node,
+ D == Dir ->
+ {[All|Replaced],true};
+ ({{N,D},Suites0},{Replaced,_}) when N == Node,
+ D == Dir ->
+ Suites1 = insert_cases1(Suite,Cases,Suites0),
+ {[{{N,D},Suites1}|Replaced],true};
+ (T,{Replaced,Match}) ->
+ {[T|Replaced],Match}
+ end, {[],false}, Tests),
+ if not Done ->
+ Tests ++ [{{Node,Dir},[{Suite,Cases}]}];
+ true ->
+ Tests1
end;
insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) ->
insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests).
@@ -1132,15 +1281,23 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when
append({{Node,Dir},Suites1},Tests);
skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
- Suites =
- case lists:keysearch({Node,Dir},1,Tests) of
- {value,{{Node,Dir},Suites0}} ->
- Suites0;
- false ->
- []
- end,
- Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites),
- insert_in_order({{Node,Dir},Suites1},Tests);
+ {Tests1,Done} =
+ lists:foldr(fun({{N,D},Suites0},{Replaced,_}) when N == Node,
+ D == Dir ->
+ Suites1 = skip_groups1(Suite,
+ [{Gr,Cases} || Gr <- Groups],
+ Cmt,Suites0),
+ {[{{N,D},Suites1}|Replaced],true};
+ (T,{Replaced,Match}) ->
+ {[T|Replaced],Match}
+ end, {[],false}, Tests),
+ if not Done ->
+ Tests ++ [{{Node,Dir},skip_groups1(Suite,
+ [{Gr,Cases} || Gr <- Groups],
+ Cmt,[])}];
+ true ->
+ Tests1
+ end;
skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests)
when is_atom(Case) ->
Cases = if Case == all -> all; true -> [Case] end,
@@ -1162,15 +1319,19 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) ->
Suites1 = skip_cases1(Suite,Cases,Cmt,[]),
append({{Node,Dir},Suites1},Tests);
skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) ->
- Suites =
- case lists:keysearch({Node,Dir},1,Tests) of
- {value,{{Node,Dir},Suites0}} ->
- Suites0;
- false ->
- []
- end,
- Suites1 = skip_cases1(Suite,Cases,Cmt,Suites),
- insert_in_order({{Node,Dir},Suites1},Tests);
+ {Tests1,Done} =
+ lists:foldr(fun({{N,D},Suites0},{Replaced,_}) when N == Node,
+ D == Dir ->
+ Suites1 = skip_cases1(Suite,Cases,Cmt,Suites0),
+ {[{{N,D},Suites1}|Replaced],true};
+ (T,{Replaced,Match}) ->
+ {[T|Replaced],Match}
+ end, {[],false}, Tests),
+ if not Done ->
+ Tests ++ [{{Node,Dir},skip_cases1(Suite,Cases,Cmt,[])}];
+ true ->
+ Tests1
+ end;
skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) ->
skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests).
@@ -1261,6 +1422,7 @@ is_node([],_) ->
valid_terms() ->
[
{define,3},
+ {specs,3},
{node,3},
{cover,2},
{cover,3},
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index d469d03e04..760cc20410 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -40,6 +40,7 @@ MODULES= \
ct_repeat_1_SUITE \
ct_testspec_1_SUITE \
ct_testspec_2_SUITE \
+ ct_testspec_3_SUITE \
ct_skip_SUITE \
ct_error_SUITE \
ct_test_server_if_1_SUITE \
diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl
index b89d7d4dc5..0f336d2d79 100644
--- a/lib/common_test/test/ct_master_SUITE.erl
+++ b/lib/common_test/test/ct_master_SUITE.erl
@@ -109,7 +109,7 @@ ct_master_test(Config) when is_list(Config) ->
ERPid = ct_test_support:start_event_receiver(Config),
- [{TSFile,ok}] = run_test(ct_master_test, FileName, Config),
+ [{[TSFile],ok}] = run_test(ct_master_test, FileName, Config),
Events = ct_test_support:get_events(ERPid, Config),
@@ -195,12 +195,12 @@ get_log_dir(_,PrivDir,NodeName) ->
run_test(_Name, FileName, Config) ->
%% run the test twice, using different html versions
- [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]},
- [{ct_master,basic_html,[true]}],
- Config),
- [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]},
- [{ct_master,basic_html,[false]}],
- Config).
+ [{[FileName],ok}] = ct_test_support:run({ct_master,run,[FileName]},
+ [{ct_master,basic_html,[true]}],
+ Config),
+ [{[FileName],ok}] = ct_test_support:run({ct_master,run,[FileName]},
+ [{ct_master,basic_html,[false]}],
+ Config).
reformat(Events, EH) ->
ct_test_support:reformat(Events, EH).
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 717f79b116..7c33fd404d 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -1132,6 +1132,8 @@ reformat([{_EH,#event{name=test_start,data=_}} | Events], EH) ->
[{EH,test_start,{'DEF',{'START_TIME','LOGDIR'}}} | reformat(Events, EH)];
reformat([{_EH,#event{name=test_done,data=_}} | Events], EH) ->
[{EH,test_done,{'DEF','STOP_TIME'}} | reformat(Events, EH)];
+reformat([{_EH,#event{name=tc_logfile,data=_}} | Events], EH) ->
+ reformat(Events, EH);
reformat([{_EH,#event{name=test_stats,data=Data}} | Events], EH) ->
[{EH,test_stats,Data} | reformat(Events, EH)];
%% use this to only print the last test_stats event:
diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl
index 9d2dc84ad3..ea22312a7d 100644
--- a/lib/common_test/test/ct_testspec_2_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_2_SUITE.erl
@@ -479,7 +479,7 @@ multiple_specs(_Config) ->
"multiple_specs.1.spec"),
SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir,
"multiple_specs.2.spec"),
- FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2],
+ FileResult = ct_testspec:collect_tests_from_file([[SpecFile1,SpecFile2]],
false),
ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
@@ -490,7 +490,7 @@ multiple_specs(_Config) ->
[{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]],
LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
- Verify = #testspec{merge_tests = false,
+ Verify = #testspec{merge_tests = true,
spec_dir = SpecDir,
nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
alias = [{to1,TO1V},{to2,TO2V}],
@@ -524,7 +524,7 @@ multiple_specs(_Config) ->
%%%
misc_config_terms(_Config) ->
CfgDir = "../cfgs/to1",
-
+ TODir = "../tests/to1",
Spec =
[{node,x,n1@h1},{node,y,n2@h2},
@@ -554,7 +554,9 @@ misc_config_terms(_Config) ->
{create_priv_dir,[auto_per_tc]},
{create_priv_dir,n1@h1,[manual_per_tc]},
- {create_priv_dir,n2@h2,[auto_per_run]}
+ {create_priv_dir,n2@h2,[auto_per_run]},
+
+ {suites,n1@h1,TODir,[x_SUITE]}
],
{ok,SpecDir} = file:get_cwd(),
@@ -599,7 +601,9 @@ misc_config_terms(_Config) ->
{n2@h2,CSS2}],
create_priv_dir = [{Node,[auto_per_tc]},
{n1@h1,[manual_per_tc]},
- {n2@h2,[auto_per_run]}]
+ {n2@h2,[auto_per_run]}],
+ tests = [{{n1@h1,get_absdir(filename:join(SpecDir,TODir))},
+ [{x_SUITE,[all]}]}]
},
verify_result(Verify,ListResult,FileResult).
@@ -688,10 +692,10 @@ define_names_1(_Config) ->
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
-verify_result(Verify,ListResult,FileResult) ->
+verify_result(VerificationRec,ListResult,FileResult) ->
{_,TSLTuples} = rec2proplist(ListResult),
{_,TSFTuples} = rec2proplist(FileResult),
- {_,VTuples} = rec2proplist(Verify),
+ {_,VTuples} = rec2proplist(VerificationRec),
VResult =
(catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) ->
{TSL,TSF};
@@ -720,6 +724,8 @@ read_config(S) ->
rec2proplist(E={error,_What}) ->
exit({invalid_testspec_record,E});
+rec2proplist([{Specs,Rec}]) when is_list(Specs) ->
+ rec2proplist(Rec);
rec2proplist(Rec) ->
[RecName|RecList] = tuple_to_list(Rec),
FieldNames =
diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl
new file mode 100644
index 0000000000..8b84b563ab
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE.erl
@@ -0,0 +1,749 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_testspec_1_SUITE
+%%%
+%%% Description:
+%%% Test test specifications
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_testspec_3_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Config1 = ct_test_support:init_per_suite(Config),
+ SpecsDir1 = filename:join(DataDir, "specs1"),
+ SpecsDir2 = filename:join(DataDir, "specs2"),
+ [{specs_dir1,SpecsDir1},{specs_dir2,SpecsDir2} | Config1].
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [start_separate,
+ start_join,
+ incl_separate1,
+ incl_separate2,
+ incl_join1,
+ incl_join2,
+ incl_both1,
+ incl_both2,
+ incl_both_and_join1,
+ incl_both_and_join2,
+ rec_incl_separate1,
+ rec_incl_separate2,
+ rec_incl_join1,
+ rec_incl_join2,
+ rec_incl_separate_join1,
+ rec_incl_separate_join2,
+ rec_incl_join_separate1,
+ rec_incl_join_separate2
+ ].
+
+groups() ->
+ [].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+
+start_separate(Config) ->
+ Specs = [fname(specs_dir1, "flat_spec1", Config),
+ fname(specs_dir2, "flat_spec2", Config)],
+ setup_and_execute(start_separate, Specs, [], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+start_join(Config) ->
+ Specs = [fname(specs_dir1, "flat_spec1", Config),
+ fname(specs_dir2, "flat_spec2", Config)],
+ setup_and_execute(start_join, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_separate1(Config) ->
+ Specs = [fname(specs_dir1, "spec_sep1", Config),
+ fname(specs_dir2, "spec_sep2", Config)],
+ setup_and_execute(incl_separate1, Specs, [], Config).
+
+incl_separate2(Config) ->
+ Specs = [fname(specs_dir1, "spec_sep1", Config),
+ fname(specs_dir2, "spec_sep2", Config)],
+ setup_and_execute(incl_separate2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_join1(Config) ->
+ Specs = [fname(specs_dir1, "spec_join1", Config),
+ fname(specs_dir2, "spec_join2", Config)],
+ setup_and_execute(incl_join1, Specs, [], Config).
+
+incl_join2(Config) ->
+ Specs = [fname(specs_dir1, "spec_join1", Config),
+ fname(specs_dir2, "spec_join2", Config)],
+ setup_and_execute(incl_join2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_both1(Config) ->
+ Specs = [fname(specs_dir1, "spec_both1", Config),
+ fname(specs_dir2, "spec_both2", Config)],
+ setup_and_execute(incl_both1, Specs, [], Config).
+
+incl_both2(Config) ->
+ Specs = [fname(specs_dir1, "spec_both1", Config),
+ fname(specs_dir2, "spec_both2", Config)],
+ setup_and_execute(incl_both2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_both_and_join1(Config) ->
+ Specs = [fname(specs_dir1, "spec_both_join1", Config),
+ fname(specs_dir2, "spec_both_join2", Config)],
+ setup_and_execute(incl_both_and_join1, Specs, [], Config).
+
+incl_both_and_join2(Config) ->
+ Specs = [fname(specs_dir1, "spec_both_join1", Config),
+ fname(specs_dir2, "spec_both_join2", Config)],
+ setup_and_execute(incl_both_and_join2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_separate1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep1", Config),
+ fname(specs_dir2, "rec_spec_sep2", Config)],
+ setup_and_execute(rec_incl_separate1, Specs, [], Config).
+
+rec_incl_separate2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep1", Config),
+ fname(specs_dir2, "rec_spec_sep2", Config)],
+ setup_and_execute(rec_incl_separate2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_join1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join1", Config),
+ fname(specs_dir2, "rec_spec_join2", Config)],
+ setup_and_execute(rec_incl_join1, Specs, [], Config).
+
+rec_incl_join2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join1", Config),
+ fname(specs_dir2, "rec_spec_join2", Config)],
+ setup_and_execute(rec_incl_join2, Specs, [{join_specs,true}], Config).
+
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_separate_join1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep_join1", Config),
+ fname(specs_dir2, "rec_spec_sep_join2", Config)],
+ setup_and_execute(rec_incl_separate_join1, Specs, [], Config).
+
+rec_incl_separate_join2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep_join1", Config),
+ fname(specs_dir2, "rec_spec_sep_join2", Config)],
+ setup_and_execute(rec_incl_separate_join2, Specs,
+ [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_join_separate1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join_sep1", Config),
+ fname(specs_dir2, "rec_spec_join_sep2", Config)],
+ setup_and_execute(rec_incl_join_separate1, Specs, [], Config).
+
+rec_incl_join_separate2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join_sep1", Config),
+ fname(specs_dir2, "rec_spec_join_sep2", Config)],
+ setup_and_execute(rec_incl_join_separate2, Specs,
+ [{join_specs,true}], Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+fname(Tag, File, Config) ->
+ filename:join(?config(Tag, Config), File).
+
+check_parameter(TCID) ->
+ {ok,{config,TCID}}.
+
+read_config(TCID) ->
+ {ok,[{tcname,list_to_atom(TCID)}]}.
+
+setup_and_execute(TCName, Specs, TestOpts, Config) ->
+
+ TestID = {userconfig,{?MODULE,atom_to_list(TCName)}},
+ TestTerms = [TestID,{spec,Specs},{label,TCName}] ++ TestOpts,
+
+ {Opts,ERPid} = setup(TestTerms, Config),
+
+ case ct_test_support:run(Opts, Config) of
+ ok ->
+ ok;
+ Error ->
+ ct:pal("Error executing with opts: ~p", [Opts]),
+ exit(Error)
+ end,
+
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(TCName,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(TCName),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+setup(Test, Config) when is_tuple(Test) ->
+ setup([Test], Config);
+setup(Tests, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ Tests ++ [{event_handler,{?eh,EvHArgs}}],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+%reformat(Events, _EH) ->
+% Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(start_separate) ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(start_join) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_separate1) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,5}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_separate2) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,5}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_join1) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_join2) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{t11_SUITE,ok_tc}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_both1) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_both2) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_both_and_join1) -> [];
+test_events(incl_both_and_join2) -> [];
+test_events(rec_incl_separate1) -> [];
+test_events(rec_incl_separate2) -> [];
+test_events(rec_incl_join1) -> [];
+test_events(rec_incl_join2) -> [];
+test_events(rec_incl_separate_join1) -> [];
+test_events(rec_incl_separate_join2) -> [];
+test_events(rec_incl_join_separate1) -> [];
+test_events(rec_incl_join_separate2) -> [];
+
+test_events(_) ->
+ [].
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11
new file mode 100644
index 0000000000..bc672568da
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11
@@ -0,0 +1 @@
+{file, cfg11}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12
new file mode 100644
index 0000000000..30f2cf6857
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12
@@ -0,0 +1 @@
+{file, cfg12}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13
new file mode 100644
index 0000000000..1860ec78e5
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13
@@ -0,0 +1 @@
+{file, cfg13}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21 b/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21
new file mode 100644
index 0000000000..b18d35443e
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21
@@ -0,0 +1 @@
+{file, cfg21}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1
new file mode 100644
index 0000000000..eff87222ea
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1
@@ -0,0 +1,4 @@
+{config, "../config1/cfg11"}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests2", t21_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1
new file mode 100644
index 0000000000..a3387f48a3
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1
@@ -0,0 +1,2 @@
+{specs,join,"spec_join1"}.
+{specs,join,"../specs2/spec_join2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1
new file mode 100644
index 0000000000..fe127eb4b9
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1
@@ -0,0 +1,5 @@
+{specs,join,"spec_sep1"}.
+{specs,join,"../specs2/spec_sep2"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1
new file mode 100644
index 0000000000..c778aa68a6
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1
@@ -0,0 +1,2 @@
+{specs,separate,"spec_sep1"}.
+{specs,separate,"../specs2/spec_sep2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1
new file mode 100644
index 0000000000..7cb5a05fff
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1
@@ -0,0 +1,2 @@
+{specs,separate,"spec_join1"}.
+{specs,separate,"../specs2/spec_join2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1
new file mode 100644
index 0000000000..46111614dc
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1
@@ -0,0 +1,2 @@
+{specs, join, "../specs1/flat_spec1"}.
+{specs, separate, "../specs2/flat_spec2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1
new file mode 100644
index 0000000000..f52b3ed030
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1
@@ -0,0 +1,6 @@
+{specs, join, "../specs1/flat_spec1"}.
+{specs, separate, "../specs2/flat_spec2"}.
+{merge_tests,false}.
+{config, "../config1/cfg12"}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests2", t21_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1
new file mode 100644
index 0000000000..baaaf35be4
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1
@@ -0,0 +1 @@
+{specs, join, ["../specs2/flat_spec2", "flat_spec1"]}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1
new file mode 100644
index 0000000000..89456c35e0
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1
@@ -0,0 +1,3 @@
+{specs, separate, "../specs2/flat_spec2"}.
+{specs, separate, "flat_spec1"}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2
new file mode 100644
index 0000000000..758d1e2514
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2
@@ -0,0 +1,5 @@
+{merge_tests, false}.
+{config, "../config2/cfg21"}.
+{suites, "../tests1", t12_SUITE}.
+{suites, "../tests1", t12_SUITE}.
+{suites, "../tests2", t22_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2
new file mode 100644
index 0000000000..19d3a3d8e2
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2
@@ -0,0 +1,5 @@
+{specs,join,"spec_join2"}.
+{specs,join,"../specs1/spec_join1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2
new file mode 100644
index 0000000000..930e68c847
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2
@@ -0,0 +1,5 @@
+{specs,join,"spec_sep2"}.
+{specs,join,"../specs1/spec_sep1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2
new file mode 100644
index 0000000000..5026f329a7
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2
@@ -0,0 +1,5 @@
+{specs,separate,"spec_sep2"}.
+{specs,separate,"../specs1/spec_sep1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2
new file mode 100644
index 0000000000..17057088b4
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2
@@ -0,0 +1,5 @@
+{specs,separate,"spec_join2"}.
+{specs,separate,"../specs1/spec_join1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2
new file mode 100644
index 0000000000..4c83115d23
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2
@@ -0,0 +1,4 @@
+{specs, separate, "../specs1/flat_spec1"}.
+{specs, join, "../specs2/flat_spec2"}.
+{config, "../config1/cfg12"}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2
new file mode 100644
index 0000000000..ad81bfb4cc
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2
@@ -0,0 +1,9 @@
+{merge_tests,true}.
+{config, "../config1/cfg12"}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests2", t21_SUITE}.
+{suites, "../tests1", t12_SUITE}.
+{suites, "../tests2", t22_SUITE}.
+
+{specs, separate, "../specs1/flat_spec1"}.
+{specs, join, "../specs2/flat_spec2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2
new file mode 100644
index 0000000000..d652dbd78f
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2
@@ -0,0 +1,5 @@
+{specs, join, ["../specs1/flat_spec1"]}.
+{specs, join, ["flat_spec2"]}.
+{config, "../config1/cfg12"}.
+{suites, "../tests2", t22_SUITE}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2
new file mode 100644
index 0000000000..8d37f508b8
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2
@@ -0,0 +1,5 @@
+{specs, separate, "../specs1/flat_spec1"}.
+{specs, separate, "flat_spec2"}.
+{config, "../config1/cfg12"}.
+{suites, "../tests2", t22_SUITE}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl
new file mode 100644
index 0000000000..bbe79ed3fe
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t11_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,1}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg11]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg11]} -> ok;
+ {incl_separate2,[cfg11]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg11]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11]} -> ok;
+ _ -> ok
+
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl
new file mode 100644
index 0000000000..810298d348
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t12_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg21]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg21]} -> ok;
+ {incl_separate2,[cfg21]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg21]} -> ok;
+ {incl_both1,[cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg21]} -> ok;
+ _ -> ok
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl
new file mode 100644
index 0000000000..9348cd8caf
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl
@@ -0,0 +1,174 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t21_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,1}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg11]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg11]} -> ok;
+ {incl_separate2,[cfg11]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg11]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11]} -> ok;
+ _ -> ok
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl
new file mode 100644
index 0000000000..a92018ec70
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl
@@ -0,0 +1,177 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t22_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg21]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg12]} -> ok;
+ {incl_separate1,[cfg21]} -> ok;
+ {incl_separate2,[cfg12]} -> ok;
+ {incl_separate2,[cfg21]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg21]} -> ok;
+ {incl_both1,[cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg21]} -> ok;
+ _ -> ok
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl
new file mode 100644
index 0000000000..d01fac3144
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl
@@ -0,0 +1,158 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t23_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+