diff options
Diffstat (limited to 'lib/common_test')
52 files changed, 3876 insertions, 620 deletions
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index a3b3f927eb..081adeaec7 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -32,7 +32,7 @@ <rev>PA1</rev> <file>common_test_app.sgml</file> </header> - <module>common_test</module> + <module since="">common_test</module> <modulesummary>A framework for automated testing of any target nodes.</modulesummary> <description> @@ -68,18 +68,19 @@ <funcs> <func> - <name>Module:all() -> Tests | {skip,Reason} </name> + <name since="">Module:all() -> Tests | {skip,Reason} </name> <fsummary>Returns the list of all test case groups and test cases in the module.</fsummary> <type> - <v>Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v> + <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v> <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> <v>GroupName = atom()</v> - <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | default</v> + <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] | default</v> <v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v> <v>Shuffle = shuffle | {shuffle,Seed}</v> <v>Seed = {integer(),integer(),integer()}</v> - <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> <v>N = integer() | forever</v> <v>Reason = term()</v> </type> @@ -91,7 +92,8 @@ test suite module to be executed. This list also specifies the order the cases and groups are executed by <c>Common Test</c>. A test case is represented by an atom, - the name of the test case function. A test case group is + the name of the test case function, or a <c>testcase</c> tuple + indicating that the test case shall be repeated. A test case group is represented by a <c>group</c> tuple, where <c>GroupName</c>, an atom, is the name of the group (defined in <seealso marker="#Module:groups-0"><c>groups/0</c></seealso>). @@ -115,18 +117,19 @@ </func> <func> - <name>Module:groups() -> GroupDefs</name> + <name since="">Module:groups() -> GroupDefs</name> <fsummary>Returns a list of test case group definitions.</fsummary> <type> <v>GroupDefs = [Group]</v> <v>Group = {GroupName,Properties,GroupsAndTestCases}</v> <v>GroupName = atom()</v> - <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}]</v> - <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v> + <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v> + <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v> <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> <v>Shuffle = shuffle | {shuffle,Seed}</v> <v>Seed = {integer(),integer(),integer()}</v> - <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> <v>N = integer() | forever</v> </type> @@ -140,7 +143,7 @@ </func> <func> - <name>Module:suite() -> [Info] </name> + <name since="">Module:suite() -> [Info] </name> <fsummary>Test suite info function (providing default data for the suite).</fsummary> <type> @@ -213,7 +216,7 @@ </func> <func> - <name>Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | + <name since="">Module:init_per_suite(Config) -> NewConfig | {skip,Reason} | {skip_and_save,Reason,SaveConfig}</name> <fsummary>Test suite initializations.</fsummary> <type> @@ -248,7 +251,7 @@ </func> <func> - <name>Module:end_per_suite(Config) -> term() | + <name since="">Module:end_per_suite(Config) -> term() | {save_config,SaveConfig}</name> <fsummary>Test suite finalization.</fsummary> <type> @@ -272,7 +275,7 @@ </func> <func> - <name>Module:group(GroupName) -> [Info] </name> + <name since="OTP R15B">Module:group(GroupName) -> [Info] </name> <fsummary>Test case group information function (providing default data for a test case group, that is, its test cases and subgroups).</fsummary> @@ -352,7 +355,7 @@ </func> <func> - <name>Module:init_per_group(GroupName, Config) -> NewConfig | + <name since="">Module:init_per_group(GroupName, Config) -> NewConfig | {skip,Reason}</name> <fsummary>Test case group initializations.</fsummary> <type> @@ -390,7 +393,7 @@ </func> <func> - <name>Module:end_per_group(GroupName, Config) -> term() | + <name since="">Module:end_per_group(GroupName, Config) -> term() | {return_group_result,Status}</name> <fsummary>Test case group finalization.</fsummary> <type> @@ -424,7 +427,7 @@ </func> <func> - <name>Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail,Reason} | {skip,Reason}</name> + <name since="">Module:init_per_testcase(TestCase, Config) -> NewConfig | {fail,Reason} | {skip,Reason}</name> <fsummary>Test case initializations.</fsummary> <type> <v> TestCase = atom()</v> @@ -454,7 +457,7 @@ </func> <func> - <name>Module:end_per_testcase(TestCase, Config) -> term() | {fail,Reason} | {save_config,SaveConfig}</name> + <name since="">Module:end_per_testcase(TestCase, Config) -> term() | {fail,Reason} | {save_config,SaveConfig}</name> <fsummary>Test case finalization.</fsummary> <type> <v>TestCase = atom()</v> @@ -486,7 +489,7 @@ </func> <func> - <name>Module:Testcase() -> [Info] </name> + <name since="OTP R14B">Module:Testcase() -> [Info] </name> <fsummary>Test case information function.</fsummary> <type> <v>Info = {timetrap,Time} | {require,Required} | {require,Name,Required} | {userdata,UserData} | {silent_connections,Conns}</v> @@ -560,7 +563,7 @@ </func> <func> - <name>Module:Testcase(Config) -> term() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() </name> + <name since="OTP R14B">Module:Testcase(Config) -> term() | {skip,Reason} | {comment,Comment} | {save_config,SaveConfig} | {skip_and_save,Reason,SaveConfig} | exit() </name> <fsummary>A test case.</fsummary> <type> <v>Config = SaveConfig = [{Key,Value}]</v> diff --git a/lib/common_test/doc/src/ct.xml b/lib/common_test/doc/src/ct.xml index c0380c4142..83c0ecb309 100644 --- a/lib/common_test/doc/src/ct.xml +++ b/lib/common_test/doc/src/ct.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct.xml</file> </header> - <module>ct</module> + <module since="">ct</module> <modulesummary>Main user interface for the Common Test framework.</modulesummary> <description> @@ -139,7 +139,7 @@ <funcs> <func> - <name>abort_current_testcase(Reason) -> ok | {error, ErrorReason}</name> + <name since="">abort_current_testcase(Reason) -> ok | {error, ErrorReason}</name> <fsummary>Aborts the currently executing test case.</fsummary> <type> <v>Reason = term()</v> @@ -157,7 +157,7 @@ </func> <func> - <name>add_config(Callback, Config) -> ok | {error, Reason}</name> + <name since="OTP R14B">add_config(Callback, Config) -> ok | {error, Reason}</name> <fsummary>Loads configuration variables using the specified callback module and configuration string.</fsummary> <type> @@ -176,7 +176,7 @@ </func> <func> - <name>break(Comment) -> ok | {error, Reason}</name> + <name since="OTP R15B02">break(Comment) -> ok | {error, Reason}</name> <fsummary>Cancels any active timetrap and pause the execution of the current test case until the user calls function continue/0.</fsummary> <type> @@ -206,7 +206,7 @@ </func> <func> - <name>break(TestCase, Comment) -> ok | {error, Reason}</name> + <name since="OTP R15B02">break(TestCase, Comment) -> ok | {error, Reason}</name> <fsummary>Works the same way as break/1, only argument TestCase makes it possible to pause a test case executing in a parallel group.</fsummary> <type> @@ -228,7 +228,7 @@ </func> <func> - <name>capture_get() -> ListOfStrings</name> + <name since="OTP R15B">capture_get() -> ListOfStrings</name> <fsummary>Equivalent to capture_get([default]).</fsummary> <type> <v>ListOfStrings = [string()]</v> @@ -240,7 +240,7 @@ </func> <func> - <name>capture_get(ExclCategories) -> ListOfStrings</name> + <name since="OTP R15B">capture_get(ExclCategories) -> ListOfStrings</name> <fsummary>Returns and purges the list of text strings buffered during the latest session of capturing printouts to stdout.</fsummary> <type> @@ -262,7 +262,7 @@ </func> <func> - <name>capture_start() -> ok</name> + <name since="OTP R15B">capture_start() -> ok</name> <fsummary>Starts capturing all text strings printed to stdout during execution of the test case.</fsummary> <desc><marker id="capture_start-0"/> @@ -276,7 +276,7 @@ </func> <func> - <name>capture_stop() -> ok</name> + <name since="OTP R15B">capture_stop() -> ok</name> <fsummary>Stops capturing text strings (a session started with capture_start/0).</fsummary> <desc><marker id="capture_stop-0"/> @@ -290,7 +290,7 @@ </func> <func> - <name>comment(Comment) -> ok</name> + <name since="">comment(Comment) -> ok</name> <fsummary>Prints the specified Comment in the comment field in the table on the test suite result page.</fsummary> <type> @@ -307,7 +307,7 @@ </func> <func> - <name>comment(Format, Args) -> ok</name> + <name since="OTP R15B">comment(Format, Args) -> ok</name> <fsummary>Prints the formatted string in the comment field in the table on the test suite result page.</fsummary> <type> @@ -326,7 +326,7 @@ </func> <func> - <name>continue() -> ok</name> + <name since="OTP R15B02">continue() -> ok</name> <fsummary>This function must be called to continue after a test case (not executing in a parallel group) has called break/1.</fsummary> <desc><marker id="continue-0"/> @@ -337,7 +337,7 @@ </func> <func> - <name>continue(TestCase) -> ok</name> + <name since="OTP R15B02">continue(TestCase) -> ok</name> <fsummary>This function must be called to continue after a test case has called break/2.</fsummary> <type> @@ -353,7 +353,7 @@ </func> <func> - <name>decrypt_config_file(EncryptFileName, TargetFileName) -> ok | {error, Reason}</name> + <name since="">decrypt_config_file(EncryptFileName, TargetFileName) -> ok | {error, Reason}</name> <fsummary>Decrypts EncryptFileName, previously generated with encrypt_config_file/2,3.</fsummary> <type> @@ -372,7 +372,7 @@ </func> <func> - <name>decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) -> ok | {error, Reason}</name> + <name since="">decrypt_config_file(EncryptFileName, TargetFileName, KeyOrFile) -> ok | {error, Reason}</name> <fsummary>Decrypts EncryptFileName, previously generated with encrypt_config_file/2,3.</fsummary> <type> @@ -390,7 +390,7 @@ </func> <func> - <name>encrypt_config_file(SrcFileName, EncryptFileName) -> ok | {error, Reason}</name> + <name since="">encrypt_config_file(SrcFileName, EncryptFileName) -> ok | {error, Reason}</name> <fsummary>Encrypts the source configuration file with DES3 and saves the result in file EncryptFileName.</fsummary> <type> @@ -416,7 +416,7 @@ </func> <func> - <name>encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> ok | {error, Reason}</name> + <name since="">encrypt_config_file(SrcFileName, EncryptFileName, KeyOrFile) -> ok | {error, Reason}</name> <fsummary>Encrypts the source configuration file with DES3 and saves the result in the target file EncryptFileName.</fsummary> <type> @@ -442,7 +442,7 @@ </func> <func> - <name>fail(Reason) -> ok</name> + <name since="">fail(Reason) -> ok</name> <fsummary>Terminates a test case with the specified error Reason.</fsummary> <type> @@ -454,7 +454,7 @@ </func> <func> - <name>fail(Format, Args) -> ok</name> + <name since="OTP R15B">fail(Format, Args) -> ok</name> <fsummary>Terminates a test case with an error message specified by a format string and a list of values (used as arguments to io_lib:format/2).</fsummary> @@ -470,7 +470,7 @@ </func> <func> - <name>get_config(Required) -> Value</name> + <name since="">get_config(Required) -> Value</name> <fsummary>Equivalent to get_config(Required, undefined, []).</fsummary> <desc><marker id="get_config-1"/> <p>Equivalent to <seealso marker="#get_config-3"><c>ct:get_config(Required, @@ -479,7 +479,7 @@ </func> <func> - <name>get_config(Required, Default) -> Value</name> + <name since="">get_config(Required, Default) -> Value</name> <fsummary>Equivalent to get_config(Required, Default, []).</fsummary> <desc><marker id="get_config-2"/> <p>Equivalent to <seealso marker="#get_config-3"><c>ct:get_config(Required, @@ -488,7 +488,7 @@ </func> <func> - <name>get_config(Required, Default, Opts) -> ValueOrElement</name> + <name since="">get_config(Required, Default, Opts) -> ValueOrElement</name> <fsummary>Reads configuration data values.</fsummary> <type> <v>Required = KeyOrName | {KeyOrName, SubKey} | {KeyOrName, SubKey, SubKey}</v> @@ -554,7 +554,7 @@ </func> <func> - <name>get_event_mgr_ref() -> EvMgrRef</name> + <name since="OTP 17.5">get_event_mgr_ref() -> EvMgrRef</name> <fsummary>Gets a reference to the <c>Common Test</c> event manager.</fsummary> <type> <v>EvMgrRef = atom()</v> @@ -572,7 +572,7 @@ </func> <func> - <name>get_progname() -> string()</name> + <name since="OTP 21.0">get_progname() -> string()</name> <fsummary>Returns the command used to start this Erlang instance.</fsummary> <desc><marker id="get_progname-0"/> <p>Returns the command used to start this Erlang instance. @@ -582,7 +582,7 @@ </func> <func> - <name>get_status() -> TestStatus | {error, Reason} | no_tests_running</name> + <name since="">get_status() -> TestStatus | {error, Reason} | no_tests_running</name> <fsummary>Returns status of ongoing test.</fsummary> <type> <v>TestStatus = [StatusElem]</v> @@ -608,7 +608,7 @@ </func> <func> - <name>get_target_name(Handle) -> {ok, TargetName} | {error, Reason}</name> + <name since="">get_target_name(Handle) -> {ok, TargetName} | {error, Reason}</name> <fsummary>Returns the name of the target that the specified connection belongs to.</fsummary> <type> @@ -622,7 +622,7 @@ </func> <func> - <name>get_testspec_terms() -> TestSpecTerms | undefined</name> + <name since="OTP 18.0">get_testspec_terms() -> TestSpecTerms | undefined</name> <fsummary>Gets a list of all test specification terms used to configure and run this test.</fsummary> <type> @@ -636,7 +636,7 @@ </func> <func> - <name>get_testspec_terms(Tags) -> TestSpecTerms | undefined</name> + <name since="OTP 18.0">get_testspec_terms(Tags) -> TestSpecTerms | undefined</name> <fsummary>Reads one or more terms from the test specification used to configure and run this test.</fsummary> <type> @@ -663,7 +663,7 @@ </func> <func> - <name>get_timetrap_info() -> {Time, {Scaling,ScaleVal}}</name> + <name since="OTP R15B">get_timetrap_info() -> {Time, {Scaling,ScaleVal}}</name> <fsummary>Reads information about the timetrap set for the current test case.</fsummary> <type> @@ -682,7 +682,7 @@ </func> <func> - <name>get_verbosity(Category) -> Level | undefined</name> + <name since="OTP 19.1">get_verbosity(Category) -> Level | undefined</name> <fsummary>Read the verbosity level for a logging category.</fsummary> <type> <v>Category = default | atom()</v> @@ -697,7 +697,7 @@ </func> <func> - <name>install(Opts) -> ok | {error, Reason}</name> + <name since="">install(Opts) -> ok | {error, Reason}</name> <fsummary>Installs configuration files and event handlers.</fsummary> <type> <v>Opts = [Opt]</v> @@ -724,7 +724,7 @@ </func> <func> - <name>listenv(Telnet) -> [Env]</name> + <name since="">listenv(Telnet) -> [Env]</name> <fsummary>Performs command listenv on the specified Telnet connection and returns the result as a list of key-value pairs.</fsummary> <type> @@ -740,7 +740,7 @@ </func> <func> - <name>log(Format) -> ok</name> + <name since="">log(Format) -> ok</name> <fsummary>Equivalent to log(default, 50, Format, [], []).</fsummary> <desc><marker id="log-1"/> <p>Equivalent to @@ -749,7 +749,7 @@ </func> <func> - <name>log(X1, X2) -> ok</name> + <name since="">log(X1, X2) -> ok</name> <fsummary>Equivalent to log(Category, Importance, Format, FormatArgs, []).</fsummary> <type> @@ -763,7 +763,7 @@ </func> <func> - <name>log(X1, X2, X3) -> ok</name> + <name since="">log(X1, X2, X3) -> ok</name> <fsummary>Equivalent to log(Category, Importance, Format, FormatArgs, Opts).</fsummary> <type> @@ -778,7 +778,7 @@ </func> <func> - <name>log(X1, X2, X3, X4) -> ok</name> + <name since="OTP R15B02">log(X1, X2, X3, X4) -> ok</name> <fsummary>Equivalent to log(Category, Importance, Format, FormatArgs, Opts).</fsummary> <type> @@ -794,7 +794,7 @@ </func> <func> - <name>log(Category, Importance, Format, FormatArgs, Opts) -> ok</name> + <name since="OTP 18.3">log(Category, Importance, Format, FormatArgs, Opts) -> ok</name> <fsummary>Prints from a test case to the log file.</fsummary> <type> <v>Category = atom()</v> @@ -825,7 +825,7 @@ </func> <func> - <name>make_priv_dir() -> ok | {error, Reason}</name> + <name since="OTP R15B01">make_priv_dir() -> ok | {error, Reason}</name> <fsummary>If the test has been started with option create_priv_dir set to manual_per_tc, in order for the test case to use the private directory, it must first create it by calling this function.</fsummary> @@ -841,7 +841,7 @@ </func> <func> - <name>notify(Name, Data) -> ok</name> + <name since="OTP R15B02">notify(Name, Data) -> ok</name> <fsummary>Sends an asynchronous notification of type Name with Data to the <c>Common Test</c> event manager.</fsummary> <type> @@ -859,7 +859,7 @@ </func> <func> - <name>pal(Format) -> ok</name> + <name since="">pal(Format) -> ok</name> <fsummary>Equivalent to pal(default, 50, Format, [], []).</fsummary> <desc><marker id="pal-1"/> <p>Equivalent to @@ -869,7 +869,7 @@ </func> <func> - <name>pal(X1, X2) -> ok</name> + <name since="">pal(X1, X2) -> ok</name> <fsummary>Equivalent to pal(Category, Importance, Format, FormatArgs, []).</fsummary> <type> @@ -883,7 +883,7 @@ </func> <func> - <name>pal(X1, X2, X3) -> ok</name> + <name since="">pal(X1, X2, X3) -> ok</name> <fsummary>Equivalent to pal(Category, Importance, Format, FormatArgs, Opts).</fsummary> <type> @@ -898,7 +898,7 @@ </func> <func> - <name>pal(X1, X2, X3, X4) -> ok</name> + <name since="OTP R15B02">pal(X1, X2, X3, X4) -> ok</name> <fsummary>Equivalent to pal(Category, Importance, Format, FormatArgs, Opts).</fsummary> <type> @@ -914,7 +914,7 @@ </func> <func> - <name>pal(Category, Importance, Format, FormatArgs, Opts) -> ok</name> + <name since="OTP 19.2">pal(Category, Importance, Format, FormatArgs, Opts) -> ok</name> <fsummary>Prints and logs from a test case.</fsummary> <type> <v>Category = atom()</v> @@ -945,7 +945,7 @@ </func> <func> - <name>parse_table(Data) -> {Heading, Table}</name> + <name since="">parse_table(Data) -> {Heading, Table}</name> <fsummary>Parses the printout from an SQL table and returns a list of tuples.</fsummary> <type> @@ -967,7 +967,7 @@ </func> <func> - <name>print(Format) -> ok</name> + <name since="">print(Format) -> ok</name> <fsummary>Equivalent to print(default, 50, Format, [], []).</fsummary> <desc><marker id="print-1"/> <p>Equivalent to <seealso marker="#print-5"><c>ct:print(default, @@ -976,7 +976,7 @@ </func> <func> - <name>print(X1, X2) -> ok</name> + <name since="OTP R15B02">print(X1, X2) -> ok</name> <fsummary>Equivalent to print(Category, Importance, Format, FormatArgs, []).</fsummary> <type> @@ -990,7 +990,7 @@ </func> <func> - <name>print(X1, X2, X3) -> ok</name> + <name since="">print(X1, X2, X3) -> ok</name> <fsummary>Equivalent to print(Category, Importance, Format, FormatArgs, Opts).</fsummary> <type> @@ -1005,7 +1005,7 @@ </func> <func> - <name>print(X1, X2, X3, X4) -> ok</name> + <name since="OTP R15B02">print(X1, X2, X3, X4) -> ok</name> <fsummary>Equivalent to print(Category, Importance, Format, FormatArgs, Opts).</fsummary> <type> @@ -1021,7 +1021,7 @@ </func> <func> - <name>print(Category, Importance, Format, FormatArgs, Opts) -> ok</name> + <name since="OTP 19.2">print(Category, Importance, Format, FormatArgs, Opts) -> ok</name> <fsummary>Prints from a test case to the console.</fsummary> <type> <v>Category = atom()</v> @@ -1048,7 +1048,7 @@ </func> <func> - <name>reload_config(Required) -> ValueOrElement | {error, Reason}</name> + <name since="OTP R14B">reload_config(Required) -> ValueOrElement | {error, Reason}</name> <fsummary>Reloads configuration file containing specified configuration key.</fsummary> <type> @@ -1071,7 +1071,7 @@ </func> <func> - <name>remaining_test_procs() -> {TestProcs,SharedGL,OtherGLs}</name> + <name since="OTP 20.2">remaining_test_procs() -> {TestProcs,SharedGL,OtherGLs}</name> <fsummary>>This function will return the identity of test- and group leader processes that are still running at the time of this call.</fsummary> <type> @@ -1107,7 +1107,7 @@ </func> <func> - <name>remove_config(Callback, Config) -> ok</name> + <name since="OTP R14B">remove_config(Callback, Config) -> ok</name> <fsummary>Removes configuration variables (together with their aliases) that were loaded with specified callback module and configuration string.</fsummary> @@ -1124,7 +1124,7 @@ </func> <func> - <name>require(Required) -> ok | {error, Reason}</name> + <name since="">require(Required) -> ok | {error, Reason}</name> <fsummary>Checks if the required configuration is available.</fsummary> <type> <v>Required = Key | {Key, SubKeys} | {Key, SubKey, SubKeys}</v> @@ -1178,7 +1178,7 @@ </func> <func> - <name>require(Name, Required) -> ok | {error, Reason}</name> + <name since="">require(Name, Required) -> ok | {error, Reason}</name> <fsummary>Checks if the required configuration is available and gives it a name.</fsummary> <type> @@ -1237,7 +1237,7 @@ </func> <func> - <name>run(TestDirs) -> Result</name> + <name since="">run(TestDirs) -> Result</name> <fsummary>Runs all test cases in all suites in the specified directories.</fsummary> <type> @@ -1251,7 +1251,7 @@ </func> <func> - <name>run(TestDir, Suite) -> Result</name> + <name since="">run(TestDir, Suite) -> Result</name> <fsummary>Runs all test cases in the specified suite.</fsummary> <desc><marker id="run-2"/> <p>Runs all test cases in the specified suite.</p> @@ -1261,7 +1261,7 @@ </func> <func> - <name>run(TestDir, Suite, Cases) -> Result</name> + <name since="">run(TestDir, Suite, Cases) -> Result</name> <fsummary>Runs the specified test cases.</fsummary> <type> <v>TestDir = string()</v> @@ -1283,7 +1283,7 @@ </func> <func> - <name>run_test(Opts) -> Result</name> + <name since="">run_test(Opts) -> Result</name> <fsummary>Runs tests as specified by the combination of options in Opts.</fsummary> <type> @@ -1355,7 +1355,7 @@ </func> <func> - <name>run_testspec(TestSpec) -> Result</name> + <name since="">run_testspec(TestSpec) -> Result</name> <fsummary>Runs a test specified by TestSpec.</fsummary> <type> <v>TestSpec = [term()]</v> @@ -1375,7 +1375,7 @@ </func> <func> - <name>set_verbosity(Category, Level) -> ok</name> + <name since="OTP 19.1">set_verbosity(Category, Level) -> ok</name> <fsummary>Set the verbosity level for a logging category.</fsummary> <type> <v>Category = default | atom()</v> @@ -1390,7 +1390,7 @@ </func> <func> - <name>sleep(Time) -> ok</name> + <name since="OTP R14B">sleep(Time) -> ok</name> <fsummary>This function, similar to timer:sleep/1, suspends the test case for a specified time.</fsummary> <type> @@ -1412,7 +1412,7 @@ </func> <func> - <name>start_interactive() -> ok</name> + <name since="">start_interactive() -> ok</name> <fsummary>Starts <c>Common Test</c> in interactive mode.</fsummary> <desc><marker id="start_interactive-0"/> <p>Starts <c>Common Test</c> in interactive mode.</p> @@ -1440,7 +1440,7 @@ </func> <func> - <name>step(TestDir, Suite, Case) -> Result</name> + <name since="">step(TestDir, Suite, Case) -> Result</name> <fsummary>Steps through a test case with the debugger.</fsummary> <type> <v>Case = atom()</v> @@ -1453,7 +1453,7 @@ </func> <func> - <name>step(TestDir, Suite, Case, Opts) -> Result</name> + <name since="">step(TestDir, Suite, Case, Opts) -> Result</name> <fsummary>Steps through a test case with the debugger.</fsummary> <type> <v>Case = atom()</v> @@ -1470,7 +1470,7 @@ </func> <func> - <name>stop_interactive() -> ok</name> + <name since="">stop_interactive() -> ok</name> <fsummary>Exits the interactive mode.</fsummary> <desc><marker id="stop_interactive-0"/> <p>Exits the interactive mode.</p> @@ -1482,7 +1482,7 @@ </func> <func> - <name>sync_notify(Name, Data) -> ok</name> + <name since="OTP R15B02">sync_notify(Name, Data) -> ok</name> <fsummary>Sends a synchronous notification of type Name with Data to the <c>Common Test</c> event manager.</fsummary> <type> @@ -1501,7 +1501,7 @@ </func> <func> - <name>testcases(TestDir, Suite) -> Testcases | {error, Reason}</name> + <name since="">testcases(TestDir, Suite) -> Testcases | {error, Reason}</name> <fsummary>Returns all test cases in the specified suite.</fsummary> <type> <v>TestDir = string()</v> @@ -1515,7 +1515,7 @@ </func> <func> - <name>timetrap(Time) -> ok</name> + <name since="OTP R14B">timetrap(Time) -> ok</name> <fsummary>Sets a new timetrap for the running test case.</fsummary> <type> <v>Time = {hours, Hours} | {minutes, Mins} | {seconds, Secs} | Millisecs | infinity | Func</v> @@ -1539,7 +1539,7 @@ </func> <func> - <name>userdata(TestDir, Suite) -> SuiteUserData | {error, Reason}</name> + <name since="">userdata(TestDir, Suite) -> SuiteUserData | {error, Reason}</name> <fsummary>Returns any data specified with tag userdata in the list of tuples returned from Suite:suite/0.</fsummary> <type> @@ -1556,7 +1556,7 @@ </func> <func> - <name>userdata(TestDir, Suite, Case::GroupOrCase) -> TCUserData | {error, Reason}</name> + <name since="">userdata(TestDir, Suite, Case::GroupOrCase) -> TCUserData | {error, Reason}</name> <fsummary>Returns any data specified with tag userdata in the list of tuples returned from Suite:group(GroupName) or Suite:Case().</fsummary> <type> diff --git a/lib/common_test/doc/src/ct_cover.xml b/lib/common_test/doc/src/ct_cover.xml index 89d944acbe..61365d3522 100644 --- a/lib/common_test/doc/src/ct_cover.xml +++ b/lib/common_test/doc/src/ct_cover.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_cover.xml</file> </header> - <module>ct_cover</module> + <module since="">ct_cover</module> <modulesummary>Common Test framework code coverage support module. </modulesummary> @@ -47,7 +47,7 @@ <funcs> <func> - <name>add_nodes(Nodes) -> {ok, StartedNodes} | {error, Reason}</name> + <name since="">add_nodes(Nodes) -> {ok, StartedNodes} | {error, Reason}</name> <fsummary>Adds nodes to current cover test (only works if cover support is active).</fsummary> <type> @@ -67,7 +67,7 @@ </func> <func> - <name>cross_cover_analyse(Level, Tests) -> ok</name> + <name since="OTP R16B">cross_cover_analyse(Level, Tests) -> ok</name> <fsummary>Accumulates cover results over multiple tests.</fsummary> <type> <v>Level = overview | details</v> @@ -83,7 +83,7 @@ </func> <func> - <name>remove_nodes(Nodes) -> ok | {error, Reason}</name> + <name since="">remove_nodes(Nodes) -> ok | {error, Reason}</name> <fsummary>Removes nodes from the current cover test.</fsummary> <type> <v>Nodes = [atom()]</v> diff --git a/lib/common_test/doc/src/ct_ftp.xml b/lib/common_test/doc/src/ct_ftp.xml index 592c5eb05d..7ee6049486 100644 --- a/lib/common_test/doc/src/ct_ftp.xml +++ b/lib/common_test/doc/src/ct_ftp.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_ftp.xml</file> </header> - <module>ct_ftp</module> + <module since="">ct_ftp</module> <modulesummary>FTP client module (based on the FTP application).</modulesummary> <description> @@ -59,7 +59,7 @@ <funcs> <func> - <name>cd(Connection, Dir) -> ok | {error, Reason}</name> + <name since="">cd(Connection, Dir) -> ok | {error, Reason}</name> <fsummary>Changes directory on remote host.</fsummary> <type> <v>Connection = connection()</v> @@ -71,7 +71,7 @@ </func> <func> - <name>close(Connection) -> ok | {error, Reason}</name> + <name since="">close(Connection) -> ok | {error, Reason}</name> <fsummary>Closes the FTP connection.</fsummary> <type> <v>Connection = connection()</v> @@ -82,7 +82,7 @@ </func> <func> - <name>delete(Connection, File) -> ok | {error, Reason}</name> + <name since="">delete(Connection, File) -> ok | {error, Reason}</name> <fsummary>Deletes a file on remote host.</fsummary> <type> <v>Connection = connection()</v> @@ -94,7 +94,7 @@ </func> <func> - <name>get(KeyOrName, RemoteFile, LocalFile) -> ok | {error, Reason}</name> + <name since="">get(KeyOrName, RemoteFile, LocalFile) -> ok | {error, Reason}</name> <fsummary>Opens an FTP connection and fetches a file from the remote host.</fsummary> <type> @@ -122,7 +122,7 @@ </func> <func> - <name>ls(Connection, Dir) -> {ok, Listing} | {error, Reason}</name> + <name since="">ls(Connection, Dir) -> {ok, Listing} | {error, Reason}</name> <fsummary>Lists directory Dir.</fsummary> <type> <v>Connection = connection()</v> @@ -135,7 +135,7 @@ </func> <func> - <name>open(KeyOrName) -> {ok, Handle} | {error, Reason}</name> + <name since="">open(KeyOrName) -> {ok, Handle} | {error, Reason}</name> <fsummary>Opens an FTP connection to the specified node.</fsummary> <type> <v>KeyOrName = Key | Name</v> @@ -164,7 +164,7 @@ </func> <func> - <name>put(KeyOrName, LocalFile, RemoteFile) -> ok | {error, Reason}</name> + <name since="">put(KeyOrName, LocalFile, RemoteFile) -> ok | {error, Reason}</name> <fsummary>Opens an FTP connection and sends a file to the remote host.</fsummary> <type> @@ -203,7 +203,7 @@ </func> <func> - <name>recv(Connection, RemoteFile) -> ok | {error, Reason}</name> + <name since="">recv(Connection, RemoteFile) -> ok | {error, Reason}</name> <fsummary>Fetches a file over FTP.</fsummary> <desc><marker id="recv-2"/> <p>Fetches a file over FTP.</p> @@ -215,7 +215,7 @@ </func> <func> - <name>recv(Connection, RemoteFile, LocalFile) -> ok | {error, Reason}</name> + <name since="">recv(Connection, RemoteFile, LocalFile) -> ok | {error, Reason}</name> <fsummary>Fetches a file over FTP.</fsummary> <type> <v>Connection = connection()</v> @@ -230,7 +230,7 @@ </func> <func> - <name>send(Connection, LocalFile) -> ok | {error, Reason}</name> + <name since="">send(Connection, LocalFile) -> ok | {error, Reason}</name> <fsummary>Sends a file over FTP.</fsummary> <desc><marker id="send-2"/> <p>Sends a file over FTP.</p> @@ -243,7 +243,7 @@ </func> <func> - <name>send(Connection, LocalFile, RemoteFile) -> ok | {error, Reason}</name> + <name since="">send(Connection, LocalFile, RemoteFile) -> ok | {error, Reason}</name> <fsummary>Sends a file over FTP.</fsummary> <type> <v>Connection = connection()</v> @@ -258,7 +258,7 @@ </func> <func> - <name>type(Connection, Type) -> ok | {error, Reason}</name> + <name since="">type(Connection, Type) -> ok | {error, Reason}</name> <fsummary>Changes the file transfer type.</fsummary> <type> <v>Connection = connection()</v> diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index 954be0ffba..b9bc54ff63 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -32,7 +32,7 @@ <rev>PA1</rev> <file>ct_hooks.sgml</file> </header> - <module>ct_hooks</module> + <module since="OTP R14B02">ct_hooks</module> <modulesummary>A callback interface on top of Common Test.</modulesummary> <description> @@ -75,7 +75,7 @@ <funcs> <func> - <name>Module:init(Id, Opts) -> {ok, State} | {ok, State, Priority}</name> + <name since="OTP R14B02">Module:init(Id, Opts) -> {ok, State} | {ok, State, Priority}</name> <fsummary>Initiates the Common Test Hook.</fsummary> <type> <v>Id = reference() | term()</v> @@ -109,7 +109,132 @@ </func> <func> - <name>Module:pre_init_per_suite(SuiteName, InitData, CTHState) -> Result</name> + <name since="OTP @OTP-14746@">Module:post_groups(SuiteName, GroupDefs) -> NewGroupDefs</name> + <fsummary>Called after groups/0.</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>GroupDefs = NewGroupDefs = [Group]</v> + <v>Group = {GroupName,Properties,GroupsAndTestCases}</v> + <v>GroupName = atom()</v> + <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v> + <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v> + <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> + <v>Shuffle = shuffle | {shuffle,Seed}</v> + <v>Seed = {integer(),integer(),integer()}</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>N = integer() | forever</v> + </type> + <desc> + <p>OPTIONAL</p> + + <p>This function is called after + <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>. + It is used to modify the test group definitions, for + instance to add or remove groups or change group properties.</p> + + <p><c>GroupDefs</c> is what + <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso> + returned, that is, a list of group definitions.</p> + + <p><c>NewGroupDefs</c> is the possibly modified version of this list.</p> + + <p>This function is called only if the CTH is added before + <c>init_per_suite</c> is run. For details, see section + <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso> + in the User's Guide.</p> + + <p>Notice that for CTHs that are installed by means of the + <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso> + function, <c>post_groups/2</c> is called before + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + hook function. However, for CTHs that are installed by means + of the CT start flag, + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + function is called first.</p> + + <note> + <p>Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the <c>post_groups/2</c> hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.</p> + </note> + </desc> + </func> + + <func> + <name since="OTP @OTP-14746@">Module:post_all(SuiteName, Return, GroupDefs) -> NewReturn</name> + <fsummary>Called after all/0.</fsummary> + <type> + <v>SuiteName = atom()</v> + <v>Return = NewReturn = Tests | {skip,Reason}</v> + <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v> + <v>TestCase = atom()</v> + <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v> + <v>GroupName = atom()</v> + <v>Properties = GroupProperties | default</v> + <v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v> + <v>Shuffle = shuffle | {shuffle,Seed}</v> + <v>Seed = {integer(),integer(),integer()}</v> + <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v> + <v>N = integer() | forever</v> + <v>GroupDefs = NewGroupDefs = [Group]</v> + <v>Group = {GroupName,GroupProperties,GroupsAndTestCases}</v> + <v>GroupProperties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v> + <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v> + <v>Reason = term()</v> + </type> + <desc> + <p>OPTIONAL</p> + + <p>This function is called after + <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso>. + It is used to modify the set of test cases and test group to + be executed, for instance to add or remove test cases and + groups, change group properties, or even skip all tests in + the suite.</p> + + <p><c>Return</c> is what + <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso> + returned, that is, a list of test cases and groups to be + executed, or a tuple <c>{skip,Reason}</c>.</p> + + <p><c>GroupDefs</c> is what + <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso> + or the <c>post_groups/2</c> hook returned, that is, a list + of group definitions.</p> + + <p><c>NewReturn</c> is the possibly modified version of <c>Return</c>.</p> + + <p>This function is called only if the CTH is added before + <c>init_per_suite</c> is run. For details, see section + <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso> + in the User's Guide.</p> + + <p>Notice that for CTHs that are installed by means of the + <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso> + function, <c>post_all/2</c> is called before + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + hook function. However, for CTHs that are installed by means + of the CT start flag, + the <seealso marker="#Module:init-2"><c>init/2</c></seealso> + function is called first.</p> + + <note> + <p>Prior to each test execution, Common Test does a + simulated test run in order to count test suites, groups + and cases for logging purposes. This causes + the <c>post_all/3</c> hook function to always be called + twice. For this reason, side effects are best avoided in + this callback.</p> + </note> + </desc> + </func> + + <func> + <name since="OTP R14B02">Module:pre_init_per_suite(SuiteName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_suite.</fsummary> <type> <v>SuiteName = atom()</v> @@ -161,7 +286,7 @@ </func> <func> - <name>Module:post_init_per_suite(SuiteName, Config, Return, CTHState) -> Result</name> + <name since="OTP R14B02">Module:post_init_per_suite(SuiteName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_suite.</fsummary> <type> <v>SuiteName = atom()</v> @@ -208,7 +333,7 @@ </func> <func> - <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -> Result</name> + <name since="OTP 19.3">Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_group.</fsummary> <type> <v>SuiteName = atom()</v> @@ -241,7 +366,7 @@ </func> <func> - <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> + <name since="OTP 19.3">Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_group.</fsummary> <type> <v>SuiteName = atom()</v> @@ -274,7 +399,7 @@ </func> <func> - <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -> Result</name> + <name since="OTP 19.3">Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_testcase.</fsummary> <type> <v>SuiteName = atom()</v> @@ -311,7 +436,7 @@ </func> <func> - <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> + <name since="OTP 19.3">Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_testcase.</fsummary> <type> <v>SuiteName = atom()</v> @@ -344,7 +469,7 @@ </func> <func> - <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -> Result</name> + <name since="OTP 19.3">Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_testcase.</fsummary> <type> <v>SuiteName = atom()</v> @@ -380,7 +505,7 @@ </func> <func> - <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> + <name since="OTP 19.3">Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_testcase.</fsummary> <type> <v>SuiteName = atom()</v> @@ -413,7 +538,7 @@ </func> <func> - <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -> Result</name> + <name since="OTP 19.3">Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_group.</fsummary> <type> <v>SuiteName = atom()</v> @@ -446,7 +571,7 @@ </func> <func> - <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> + <name since="OTP 19.3">Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_group.</fsummary> <type> <v>SuiteName = atom()</v> @@ -479,7 +604,7 @@ </func> <func> - <name>Module:pre_end_per_suite(SuiteName, EndData, CTHState) -> Result</name> + <name since="OTP R14B02">Module:pre_end_per_suite(SuiteName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_suite.</fsummary> <type> <v>SuiteName = atom()</v> @@ -506,7 +631,7 @@ </func> <func> - <name>Module:post_end_per_suite(SuiteName, Config, Return, CTHState) -> Result</name> + <name since="OTP R14B02">Module:post_end_per_suite(SuiteName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_suite.</fsummary> <type> <v>SuiteName = atom()</v> @@ -533,7 +658,7 @@ </func> <func> - <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> + <name since="OTP 19.3">Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> <v>SuiteName = atom()</v> @@ -577,7 +702,7 @@ </func> <func> - <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> + <name since="OTP 19.3">Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> <v>SuiteName = atom()</v> @@ -623,7 +748,7 @@ </func> <func> - <name>Module:terminate(CTHState)</name> + <name since="OTP R14B02">Module:terminate(CTHState)</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> <v>CTHState = term()</v> @@ -637,7 +762,7 @@ </func> <func> - <name>Module:id(Opts) -> Id</name> + <name since="OTP R14B02">Module:id(Opts) -> Id</name> <fsummary>Called before the init function of a CTH.</fsummary> <type> <v>Opts = term()</v> diff --git a/lib/common_test/doc/src/ct_master.xml b/lib/common_test/doc/src/ct_master.xml index 6bde4644c6..2ab421fe9e 100644 --- a/lib/common_test/doc/src/ct_master.xml +++ b/lib/common_test/doc/src/ct_master.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_master.xml</file> </header> - <module>ct_master</module> + <module since="">ct_master</module> <modulesummary>Distributed test execution control for Common Test.</modulesummary> <description> @@ -46,7 +46,7 @@ <funcs> <func> - <name>abort() -> ok</name> + <name since="">abort() -> ok</name> <fsummary>Stops all running tests.</fsummary> <desc><marker id="abort-0"/> <p>Stops all running tests.</p> @@ -54,7 +54,7 @@ </func> <func> - <name>abort(Nodes) -> ok</name> + <name since="">abort(Nodes) -> ok</name> <fsummary>Stops tests on specified nodes.</fsummary> <type> <v>Nodes = atom() | [atom()]</v> @@ -65,7 +65,7 @@ </func> <func> - <name>basic_html(Bool) -> ok</name> + <name since="OTP R15B01">basic_html(Bool) -> ok</name> <fsummary>If set to true, the ct_master logs are written on a primitive HTML format, not using the <c>Common Test</c> CSS style sheet.</fsummary> <type> @@ -79,7 +79,7 @@ </func> <func> - <name>get_event_mgr_ref() -> MasterEvMgrRef</name> + <name since="OTP 17.5">get_event_mgr_ref() -> MasterEvMgrRef</name> <fsummary>Gets a reference to the <c>Common Test</c> master event manager.</fsummary> <type> @@ -98,7 +98,7 @@ </func> <func> - <name>progress() -> [{Node, Status}]</name> + <name since="">progress() -> [{Node, Status}]</name> <fsummary>Returns test progress.</fsummary> <type> <v>Node = atom()</v> @@ -112,7 +112,7 @@ </func> <func> - <name>run(TestSpecs) -> ok</name> + <name since="">run(TestSpecs) -> ok</name> <fsummary>Equivalent to run(TestSpecs, false, [], []).</fsummary> <type> <v>TestSpecs = string() | [SeparateOrMerged]</v> @@ -124,7 +124,7 @@ </func> <func> - <name>run(TestSpecs, InclNodes, ExclNodes) -> ok</name> + <name since="">run(TestSpecs, InclNodes, ExclNodes) -> ok</name> <fsummary>Equivalent to run(TestSpecs, false, InclNodes, ExclNodes). </fsummary> <type> @@ -140,7 +140,7 @@ </func> <func> - <name>run(TestSpecs, AllowUserTerms, InclNodes, ExclNodes) -> ok</name> + <name since="">run(TestSpecs, AllowUserTerms, InclNodes, ExclNodes) -> ok</name> <fsummary>Tests are spawned on the nodes as specified in TestSpecs. </fsummary> <type> @@ -162,7 +162,7 @@ </func> <func> - <name>run_on_node(TestSpecs, Node) -> ok</name> + <name since="">run_on_node(TestSpecs, Node) -> ok</name> <fsummary>Equivalent to run_on_node(TestSpecs, false, Node).</fsummary> <type> <v>TestSpecs = string() | [SeparateOrMerged]</v> @@ -177,7 +177,7 @@ </func> <func> - <name>run_on_node(TestSpecs, AllowUserTerms, Node) -> ok</name> + <name since="">run_on_node(TestSpecs, AllowUserTerms, Node) -> ok</name> <fsummary>Tests are spawned on Node according to TestSpecs.</fsummary> <type> <v>TestSpecs = string() | [SeparateOrMerged]</v> @@ -191,7 +191,7 @@ </func> <func> - <name>run_test(Node, Opts) -> ok</name> + <name since="">run_test(Node, Opts) -> ok</name> <fsummary>Tests are spawned on Node using ct:run_test/1.</fsummary> <type> <v>Node = atom()</v> diff --git a/lib/common_test/doc/src/ct_netconfc.xml b/lib/common_test/doc/src/ct_netconfc.xml index 7ec8f23073..8fbe5f3df6 100644 --- a/lib/common_test/doc/src/ct_netconfc.xml +++ b/lib/common_test/doc/src/ct_netconfc.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_netconfc.xml</file> </header> - <module>ct_netconfc</module> + <module since="OTP R15B02">ct_netconfc</module> <modulesummary>NETCONF client module.</modulesummary> <description> @@ -312,8 +312,8 @@ <funcs> <func> - <name name="action" arity="2"/> - <name name="action" arity="3"/> + <name name="action" arity="2" since="OTP R15B02"/> + <name name="action" arity="3" since="OTP R15B02"/> <fsummary>Executes an action.</fsummary> <desc> <p>Executes an action. If the return type is void, <c>ok</c> is @@ -322,8 +322,8 @@ </func> <func> - <name name="close_session" arity="1"/> - <name name="close_session" arity="2"/> + <name name="close_session" arity="1" since="OTP R15B02"/> + <name name="close_session" arity="2" since="OTP R15B02"/> <fsummary>Requests graceful termination of the session associated with the client.</fsummary> <desc> @@ -339,7 +339,7 @@ </func> <func> - <name name="connect" arity="1"/> + <name name="connect" arity="1" since="OTP 20.0"/> <fsummary>Opens an SSH connection to a NETCONF server.</fsummary> <desc> <p>Opens an SSH connection to a NETCONF server.</p> @@ -361,7 +361,7 @@ </func> <func> - <name name="connect" arity="2"/> + <name name="connect" arity="2" since="OTP 20.0"/> <fsummary>Opens an SSH connection to a named NETCONF server.</fsummary> <desc> <p>Open an SSH connection to a named NETCONF server.</p> @@ -399,8 +399,8 @@ </func> <func> - <name name="copy_config" arity="3"/> - <name name="copy_config" arity="4"/> + <name name="copy_config" arity="3" since="OTP R15B02"/> + <name name="copy_config" arity="4" since="OTP R15B02"/> <fsummary>Copies configuration data.</fsummary> <desc> <p>Copies configuration data.</p> @@ -412,12 +412,12 @@ </func> <func> - <name>create_subscription(Client) -> Result</name> - <name>create_subscription(Client, Stream) -> Result</name> - <name>create_subscription(Client, Stream, Filter) -> Result</name> - <name>create_subscription(Client, Stream, Filter, Timeout) -> Result</name> - <name name="create_subscription" arity="5" clause_i="2"/> - <name name="create_subscription" arity="6"/> + <name since="OTP R15B02">create_subscription(Client) -> Result</name> + <name since="OTP R15B02">create_subscription(Client, Stream) -> Result</name> + <name since="OTP R15B02">create_subscription(Client, Stream, Filter) -> Result</name> + <name since="OTP R15B02">create_subscription(Client, Stream, Filter, Timeout) -> Result</name> + <name name="create_subscription" arity="5" clause_i="2" since="OTP R15B02"/> + <name name="create_subscription" arity="6" since="OTP R15B02"/> <fsummary>Creates a subscription for event notifications.</fsummary> <desc> <p>Creates a subscription for event notifications.</p> @@ -490,8 +490,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="delete_config" arity="2"/> - <name name="delete_config" arity="3"/> + <name name="delete_config" arity="2" since="OTP R15B02"/> + <name name="delete_config" arity="3" since="OTP R15B02"/> <fsummary>Deletes configuration data.</fsummary> <desc> <p>Deletes configuration data.</p> @@ -502,7 +502,7 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="disconnect" arity="1"/> + <name name="disconnect" arity="1" since="OTP 20.0"/> <fsummary>Closes the given SSH connection.</fsummary> <desc> <p>Closes the given SSH connection.</p> @@ -514,10 +514,10 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="edit_config" arity="3"/> - <name name="edit_config" arity="4" clause_i="1"/> - <name name="edit_config" arity="4" clause_i="2"/> - <name name="edit_config" arity="5"/> + <name name="edit_config" arity="3" since="OTP R15B02"/> + <name name="edit_config" arity="4" clause_i="1" since="OTP 18.0"/> + <name name="edit_config" arity="4" clause_i="2" since="OTP R15B02"/> + <name name="edit_config" arity="5" since="OTP 18.0"/> <fsummary>Edits configuration data.</fsummary> <desc> <p>Edits configuration data.</p> @@ -542,8 +542,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="get" arity="2"/> - <name name="get" arity="3"/> + <name name="get" arity="2" since="OTP R15B02"/> + <name name="get" arity="3" since="OTP R15B02"/> <fsummary>Gets data.</fsummary> <desc> <p>Gets data.</p> @@ -557,8 +557,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="get_capabilities" arity="1"/> - <name name="get_capabilities" arity="2"/> + <name name="get_capabilities" arity="1" since="OTP R15B02"/> + <name name="get_capabilities" arity="2" since="OTP R15B02"/> <fsummary>Returns the server side capabilities.</fsummary> <desc> <p>Returns the server side capabilities.</p> @@ -582,8 +582,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="get_config" arity="3"/> - <name name="get_config" arity="4"/> + <name name="get_config" arity="3" since="OTP R15B02"/> + <name name="get_config" arity="4" since="OTP R15B02"/> <fsummary>Gets configuration data.</fsummary> <desc> <p>Gets configuration data.</p> @@ -597,10 +597,10 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="get_event_streams" arity="1"/> - <name name="get_event_streams" arity="2" clause_i="1"/> - <name name="get_event_streams" arity="2" clause_i="2"/> - <name name="get_event_streams" arity="3"/> + <name name="get_event_streams" arity="1" since="OTP 20.0"/> + <name name="get_event_streams" arity="2" clause_i="1" since="OTP R15B02"/> + <name name="get_event_streams" arity="2" clause_i="2" since="OTP 20.0"/> + <name name="get_event_streams" arity="3" since="OTP R15B02"/> <fsummary>Sends a request to get the specified event streams.</fsummary> <desc> <p>Sends a request to get the specified event streams.</p> @@ -637,8 +637,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="get_session_id" arity="1"/> - <name name="get_session_id" arity="2"/> + <name name="get_session_id" arity="1" since="OTP R15B02"/> + <name name="get_session_id" arity="2" since="OTP R15B02"/> <fsummary>Returns the session Id associated with the specified client.</fsummary> <desc> @@ -647,9 +647,9 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="hello" arity="1"/> - <name name="hello" arity="2"/> - <name name="hello" arity="3"/> + <name name="hello" arity="1" since="OTP R15B02"/> + <name name="hello" arity="2" since="OTP R15B02"/> + <name name="hello" arity="3" since="OTP 17.5.3"/> <fsummary>Exchanges hello messages with the server.</fsummary> <desc> <p>Exchanges <c>hello</c> messages with the server.</p> @@ -660,8 +660,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="kill_session" arity="2"/> - <name name="kill_session" arity="3"/> + <name name="kill_session" arity="2" since="OTP R15B02"/> + <name name="kill_session" arity="3" since="OTP R15B02"/> <fsummary>Forces termination of the session associated with the supplied session Id.</fsummary> <desc> @@ -682,8 +682,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="lock" arity="2"/> - <name name="lock" arity="3"/> + <name name="lock" arity="2" since="OTP R15B02"/> + <name name="lock" arity="3" since="OTP R15B02"/> <fsummary>Locks the configuration target.</fsummary> <desc> <p>Locks the configuration target.</p> @@ -703,7 +703,7 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="only_open" arity="1"/> + <name name="only_open" arity="1" since="OTP R15B02"/> <fsummary>Opens a NETCONF session, but does not send hello.</fsummary> <desc> <p>Opens a NETCONF session, but does not send <c>hello</c>.</p> @@ -714,7 +714,7 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="only_open" arity="2"/> + <name name="only_open" arity="2" since="OTP R15B02"/> <fsummary>Opens a named NETCONF session, but does not send hello.</fsummary> <desc> <p>Opens a named NETCONF session, but does not send <c>hello</c>.</p> @@ -725,7 +725,7 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="open" arity="1"/> + <name name="open" arity="1" since="OTP R15B02"/> <fsummary>Opens a NETCONF session and exchanges hello messages.</fsummary> <desc> <p>Opens a NETCONF session and exchanges <c>hello</c> messages.</p> @@ -749,7 +749,7 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="open" arity="2"/> + <name name="open" arity="2" since="OTP R15B02"/> <fsummary>Opens a named NETCONF session and exchanges hello messages.</fsummary> <desc> @@ -791,8 +791,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="send" arity="2"/> - <name name="send" arity="3"/> + <name name="send" arity="2" since="OTP R16B02"/> + <name name="send" arity="3" since="OTP R16B02"/> <fsummary>Sends an XML document to the server.</fsummary> <desc> <p>Sends an XML document to the server.</p> @@ -804,8 +804,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="send_rpc" arity="2"/> - <name name="send_rpc" arity="3"/> + <name name="send_rpc" arity="2" since="OTP R16B02"/> + <name name="send_rpc" arity="3" since="OTP R16B02"/> <fsummary>Sends a NETCONF rpc request to the server.</fsummary> <desc> <p>Sends a NETCONF <c>rpc</c> request to the server.</p> @@ -820,10 +820,10 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="session" arity="1"/> - <name name="session" arity="2" clause_i="1"/> - <name name="session" arity="2" clause_i="2"/> - <name name="session" arity="3"/> + <name name="session" arity="1" since="OTP 20.0"/> + <name name="session" arity="2" clause_i="1" since="OTP 20.0"/> + <name name="session" arity="2" clause_i="2" since="OTP 20.0"/> + <name name="session" arity="3" since="OTP 20.0"/> <fsummary>Opens a NETCONF session as a channel on the given SSH connection, and exchanges hello messages with the server.</fsummary> @@ -848,8 +848,8 @@ create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> </func> <func> - <name name="unlock" arity="2"/> - <name name="unlock" arity="3"/> + <name name="unlock" arity="2" since="OTP R15B02"/> + <name name="unlock" arity="3" since="OTP R15B02"/> <fsummary>Unlocks the configuration target.</fsummary> <desc> <p>Unlocks the configuration target.</p> diff --git a/lib/common_test/doc/src/ct_property_test.xml b/lib/common_test/doc/src/ct_property_test.xml index 028e5eb69f..1e01d9a5d7 100644 --- a/lib/common_test/doc/src/ct_property_test.xml +++ b/lib/common_test/doc/src/ct_property_test.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_property_test.xml</file> </header> - <module>ct_property_test</module> + <module since="OTP 17.3">ct_property_test</module> <modulesummary>EXPERIMENTAL support in Common Test for calling property-based tests.</modulesummary> @@ -79,7 +79,7 @@ <funcs> <func> - <name>init_per_suite(Config) -> Config | {skip, Reason}</name> + <name since="OTP 17.3">init_per_suite(Config) -> Config | {skip, Reason}</name> <fsummary>Initializes Config for property testing.</fsummary> <desc><marker id="init_per_suite-1"/> <p>Initializes <c>Config</c> for property testing.</p> @@ -98,7 +98,7 @@ </func> <func> - <name>quickcheck(Property, Config) -> true | {fail, Reason}</name> + <name since="OTP 17.3">quickcheck(Property, Config) -> true | {fail, Reason}</name> <fsummary>Calls quickcheck and returns the result in a form suitable for Common Test.</fsummary> <desc><marker id="quickcheck-2"/> diff --git a/lib/common_test/doc/src/ct_rpc.xml b/lib/common_test/doc/src/ct_rpc.xml index 90e6b833f7..00a4dcec08 100644 --- a/lib/common_test/doc/src/ct_rpc.xml +++ b/lib/common_test/doc/src/ct_rpc.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_rpc.xml</file> </header> - <module>ct_rpc</module> + <module since="">ct_rpc</module> <modulesummary>Common Test specific layer on Erlang/OTP rpc.</modulesummary> <description> @@ -43,7 +43,7 @@ <funcs> <func> - <name>app_node(App, Candidates) -> NodeName</name> + <name since="">app_node(App, Candidates) -> NodeName</name> <fsummary>From a set of candidate nodes determines which of them is running the application App.</fsummary> <type> @@ -61,7 +61,7 @@ </func> <func> - <name>app_node(App, Candidates, FailOnBadRPC) -> NodeName</name> + <name since="">app_node(App, Candidates, FailOnBadRPC) -> NodeName</name> <fsummary>Same as app_node/2, except that argument FailOnBadRPC determines if the search for a candidate node is to stop if badrpc is received at some point.</fsummary> @@ -81,7 +81,7 @@ </func> <func> - <name>app_node(App, Candidates, FailOnBadRPC, Cookie) -> NodeName</name> + <name since="">app_node(App, Candidates, FailOnBadRPC, Cookie) -> NodeName</name> <fsummary>Same as app_node/2, except that argument FailOnBadRPC determines if the search for a candidate node is to stop if badrpc is received at some point.</fsummary> @@ -105,7 +105,7 @@ </func> <func> - <name>call(Node, Module, Function, Args) -> term() | {badrpc, Reason}</name> + <name since="">call(Node, Module, Function, Args) -> term() | {badrpc, Reason}</name> <fsummary>Same as call(Node, Module, Function, Args, infinity).</fsummary> <desc><marker id="call-4"/> <p>Same as <c>call(Node, Module, Function, Args, infinity)</c>.</p> @@ -113,7 +113,7 @@ </func> <func> - <name>call(Node, Module, Function, Args, TimeOut) -> term() | {badrpc, Reason}</name> + <name since="">call(Node, Module, Function, Args, TimeOut) -> term() | {badrpc, Reason}</name> <fsummary>Evaluates apply(Module, Function, Args) on the node Node.</fsummary> <type> @@ -136,7 +136,7 @@ </func> <func> - <name>call(Node, Module, Function, Args, TimeOut, Cookie) -> term() | {badrpc, Reason}</name> + <name since="">call(Node, Module, Function, Args, TimeOut, Cookie) -> term() | {badrpc, Reason}</name> <fsummary>Evaluates apply(Module, Function, Args) on the node Node.</fsummary> <type> @@ -163,7 +163,7 @@ </func> <func> - <name>cast(Node, Module, Function, Args) -> ok</name> + <name since="">cast(Node, Module, Function, Args) -> ok</name> <fsummary>Evaluates apply(Module, Function, Args) on the node Node.</fsummary> <type> @@ -187,7 +187,7 @@ </func> <func> - <name>cast(Node, Module, Function, Args, Cookie) -> ok</name> + <name since="">cast(Node, Module, Function, Args, Cookie) -> ok</name> <fsummary>Evaluates apply(Module, Function, Args) on the node Node.</fsummary> <type> diff --git a/lib/common_test/doc/src/ct_slave.xml b/lib/common_test/doc/src/ct_slave.xml index 9d9aa50051..84e619482d 100644 --- a/lib/common_test/doc/src/ct_slave.xml +++ b/lib/common_test/doc/src/ct_slave.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_slave.xml</file> </header> - <module>ct_slave</module> + <module since="OTP R14B">ct_slave</module> <modulesummary>Common Test framework functions for starting and stopping nodes for Large-Scale Testing.</modulesummary> @@ -50,7 +50,7 @@ <funcs> <func> - <name>start(Node) -> Result</name> + <name since="OTP R14B">start(Node) -> Result</name> <fsummary>Starts an Erlang node with name Node on the local host.</fsummary> <type> @@ -68,7 +68,7 @@ </func> <func> - <name>start(HostOrNode, NodeOrOpts) -> Result</name> + <name since="OTP R14B">start(HostOrNode, NodeOrOpts) -> Result</name> <fsummary>Starts an Erlang node with default options on a specified host, or on the local host with specified options.</fsummary> <type> @@ -90,7 +90,7 @@ </func> <func> - <name>start(Host, Node, Opts) -> Result</name> + <name since="OTP R14B">start(Host, Node, Opts) -> Result</name> <fsummary>Starts an Erlang node with name Node on host Host as specified by the combination of options in Opts.</fsummary> <type> @@ -184,7 +184,7 @@ </func> <func> - <name>stop(Node) -> Result</name> + <name since="OTP R14B">stop(Node) -> Result</name> <fsummary>Stops the running Erlang node with name Node on the local host.</fsummary> <type> @@ -199,7 +199,7 @@ </func> <func> - <name>stop(Host, Node) -> Result</name> + <name since="OTP R14B">stop(Host, Node) -> Result</name> <fsummary>Stops the running Erlang node with name Node on host Host.</fsummary> <type> diff --git a/lib/common_test/doc/src/ct_snmp.xml b/lib/common_test/doc/src/ct_snmp.xml index 0a5e52b16c..343781814a 100644 --- a/lib/common_test/doc/src/ct_snmp.xml +++ b/lib/common_test/doc/src/ct_snmp.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_snmp.xml</file> </header> - <module>ct_snmp</module> + <module since="">ct_snmp</module> <modulesummary>Common Test user interface module for the SNMP application.</modulesummary> <description> @@ -240,7 +240,7 @@ <funcs> <func> - <name>get_next_values(Agent, Oids, MgrAgentConfName) -> SnmpReply</name> + <name since="">get_next_values(Agent, Oids, MgrAgentConfName) -> SnmpReply</name> <fsummary>Issues a synchronous SNMP get next request.</fsummary> <type> <v>Agent = agent_name()</v> @@ -254,7 +254,7 @@ </func> <func> - <name>get_values(Agent, Oids, MgrAgentConfName) -> SnmpReply</name> + <name since="">get_values(Agent, Oids, MgrAgentConfName) -> SnmpReply</name> <fsummary>Issues a synchronous SNMP get request.</fsummary> <type> <v>Agent = agent_name()</v> @@ -268,7 +268,7 @@ </func> <func> - <name>load_mibs(Mibs) -> ok | {error, Reason}</name> + <name since="">load_mibs(Mibs) -> ok | {error, Reason}</name> <fsummary>Loads the MIBs into agent snmp_master_agent.</fsummary> <type> <v>Mibs = [MibName]</v> @@ -281,7 +281,7 @@ </func> <func> - <name>register_agents(MgrAgentConfName, ManagedAgents) -> ok | {error, Reason}</name> + <name since="">register_agents(MgrAgentConfName, ManagedAgents) -> ok | {error, Reason}</name> <fsummary>Explicitly instructs the manager to handle this agent.</fsummary> <type> @@ -300,7 +300,7 @@ </func> <func> - <name>register_users(MgrAgentConfName, Users) -> ok | {error, Reason}</name> + <name since="">register_users(MgrAgentConfName, Users) -> ok | {error, Reason}</name> <fsummary>Registers the manager entity (=user) responsible for specific agent(s).</fsummary> <type> @@ -319,7 +319,7 @@ </func> <func> - <name>register_usm_users(MgrAgentConfName, UsmUsers) -> ok | {error, Reason}</name> + <name since="">register_usm_users(MgrAgentConfName, UsmUsers) -> ok | {error, Reason}</name> <fsummary>Explicitly instructs the manager to handle this USM user.</fsummary> <type> <v>MgrAgentConfName = atom()</v> @@ -337,7 +337,7 @@ </func> <func> - <name>set_info(Config) -> [{Agent, OldVarsAndVals, NewVarsAndVals}]</name> + <name since="">set_info(Config) -> [{Agent, OldVarsAndVals, NewVarsAndVals}]</name> <fsummary>Returns a list of all successful set requests performed in the test case in reverse order.</fsummary> <type> @@ -357,7 +357,7 @@ </func> <func> - <name>set_values(Agent, VarsAndVals, MgrAgentConfName, Config) -> SnmpReply</name> + <name since="">set_values(Agent, VarsAndVals, MgrAgentConfName, Config) -> SnmpReply</name> <fsummary>Issues a synchronous SNMP set request.</fsummary> <type> <v>Agent = agent_name()</v> @@ -372,7 +372,7 @@ </func> <func> - <name>start(Config, MgrAgentConfName) -> ok</name> + <name since="">start(Config, MgrAgentConfName) -> ok</name> <fsummary>Equivalent to start(Config, MgrAgentConfName, undefined).</fsummary> <desc><marker id="start-2"/> @@ -383,7 +383,7 @@ </func> <func> - <name>start(Config, MgrAgentConfName, SnmpAppConfName) -> ok</name> + <name since="">start(Config, MgrAgentConfName, SnmpAppConfName) -> ok</name> <fsummary>Starts an SNMP manager and/or agent.</fsummary> <type> <v>Config = [{Key, Value}]</v> @@ -415,7 +415,7 @@ </func> <func> - <name>stop(Config) -> ok</name> + <name since="">stop(Config) -> ok</name> <fsummary>Stops the SNMP manager and/or agent, and removes all files created.</fsummary> <type> @@ -430,7 +430,7 @@ </func> <func> - <name>unload_mibs(Mibs) -> ok | {error, Reason}</name> + <name since="OTP R16B">unload_mibs(Mibs) -> ok | {error, Reason}</name> <fsummary>Unloads the MIBs from agent snmp_master_agent.</fsummary> <type> <v>Mibs = [MibName]</v> @@ -443,7 +443,7 @@ </func> <func> - <name>unregister_agents(MgrAgentConfName) -> ok</name> + <name since="">unregister_agents(MgrAgentConfName) -> ok</name> <fsummary>Unregisters all managed agents.</fsummary> <type> <v>MgrAgentConfName = atom()</v> @@ -455,7 +455,7 @@ </func> <func> - <name>unregister_agents(MgrAgentConfName, ManagedAgents) -> ok</name> + <name since="OTP R16B">unregister_agents(MgrAgentConfName, ManagedAgents) -> ok</name> <fsummary>Unregisters the specified managed agents.</fsummary> <type> <v>MgrAgentConfName = atom()</v> @@ -468,7 +468,7 @@ </func> <func> - <name>unregister_users(MgrAgentConfName) -> ok</name> + <name since="">unregister_users(MgrAgentConfName) -> ok</name> <fsummary>Unregisters all users.</fsummary> <type> <v>MgrAgentConfName = atom()</v> @@ -480,7 +480,7 @@ </func> <func> - <name>unregister_users(MgrAgentConfName, Users) -> ok</name> + <name since="OTP R16B">unregister_users(MgrAgentConfName, Users) -> ok</name> <fsummary>Unregisters the specified users.</fsummary> <type> <v>MgrAgentConfName = atom()</v> @@ -493,7 +493,7 @@ </func> <func> - <name>unregister_usm_users(MgrAgentConfName) -> ok</name> + <name since="OTP R16B">unregister_usm_users(MgrAgentConfName) -> ok</name> <fsummary>Unregisters all USM users.</fsummary> <type> <v>MgrAgentConfName = atom()</v> @@ -505,7 +505,7 @@ </func> <func> - <name>unregister_usm_users(MgrAgentConfName, UsmUsers) -> ok</name> + <name since="OTP R16B">unregister_usm_users(MgrAgentConfName, UsmUsers) -> ok</name> <fsummary>Unregisters the specified USM users.</fsummary> <type> <v>MgrAgentConfName = atom()</v> diff --git a/lib/common_test/doc/src/ct_ssh.xml b/lib/common_test/doc/src/ct_ssh.xml index 0c7efed154..8d9f31aff8 100644 --- a/lib/common_test/doc/src/ct_ssh.xml +++ b/lib/common_test/doc/src/ct_ssh.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_ssh.xml</file> </header> - <module>ct_ssh</module> + <module since="">ct_ssh</module> <modulesummary>SSH/SFTP client module.</modulesummary> <description> @@ -95,7 +95,7 @@ <funcs> <func> - <name>apread(SSH, Handle, Position, Length) -> Result</name> + <name since="">apread(SSH, Handle, Position, Length) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -109,7 +109,7 @@ </func> <func> - <name>apread(SSH, Server, Handle, Position, Length) -> Result</name> + <name since="">apread(SSH, Server, Handle, Position, Length) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -123,7 +123,7 @@ </func> <func> - <name>apwrite(SSH, Handle, Position, Data) -> Result</name> + <name since="">apwrite(SSH, Handle, Position, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -137,7 +137,7 @@ </func> <func> - <name>apwrite(SSH, Server, Handle, Position, Data) -> Result</name> + <name since="">apwrite(SSH, Server, Handle, Position, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -151,7 +151,7 @@ </func> <func> - <name>aread(SSH, Handle, Len) -> Result</name> + <name since="">aread(SSH, Handle, Len) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -165,7 +165,7 @@ </func> <func> - <name>aread(SSH, Server, Handle, Len) -> Result</name> + <name since="">aread(SSH, Server, Handle, Len) -> Result</name> <fsummary>For inforamtion and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -179,7 +179,7 @@ </func> <func> - <name>awrite(SSH, Handle, Data) -> Result</name> + <name since="">awrite(SSH, Handle, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -193,7 +193,7 @@ </func> <func> - <name>awrite(SSH, Server, Handle, Data) -> Result</name> + <name since="">awrite(SSH, Server, Handle, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -207,7 +207,7 @@ </func> <func> - <name>close(SSH, Handle) -> Result</name> + <name since="">close(SSH, Handle) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -221,7 +221,7 @@ </func> <func> - <name>close(SSH, Server, Handle) -> Result</name> + <name since="">close(SSH, Server, Handle) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -235,7 +235,7 @@ </func> <func> - <name>connect(KeyOrName) -> {ok, Handle} | {error, Reason}</name> + <name since="">connect(KeyOrName) -> {ok, Handle} | {error, Reason}</name> <fsummary>Equivalent to connect(KeyOrName, host, []).</fsummary> <desc><marker id="connect-1"/> <p>Equivalent to @@ -245,7 +245,7 @@ </func> <func> - <name>connect(KeyOrName, ConnType) -> {ok, Handle} | {error, Reason}</name> + <name since="">connect(KeyOrName, ConnType) -> {ok, Handle} | {error, Reason}</name> <fsummary>Equivalent to connect(KeyOrName, ConnType, []).</fsummary> <desc><marker id="connect-2"/> <p>Equivalent to @@ -255,7 +255,7 @@ </func> <func> - <name>connect(KeyOrName, ConnType, ExtraOpts) -> {ok, Handle} | {error, Reason}</name> + <name since="">connect(KeyOrName, ConnType, ExtraOpts) -> {ok, Handle} | {error, Reason}</name> <fsummary>Opens an SSH or SFTP connection using the information associated with KeyOrName.</fsummary> <type> @@ -301,7 +301,7 @@ </func> <func> - <name>del_dir(SSH, Name) -> Result</name> + <name since="">del_dir(SSH, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -315,7 +315,7 @@ </func> <func> - <name>del_dir(SSH, Server, Name) -> Result</name> + <name since="">del_dir(SSH, Server, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -329,7 +329,7 @@ </func> <func> - <name>delete(SSH, Name) -> Result</name> + <name since="">delete(SSH, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -343,7 +343,7 @@ </func> <func> - <name>delete(SSH, Server, Name) -> Result</name> + <name since="">delete(SSH, Server, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -357,7 +357,7 @@ </func> <func> - <name>disconnect(SSH) -> ok | {error, Reason}</name> + <name since="">disconnect(SSH) -> ok | {error, Reason}</name> <fsummary>Closes an SSH/SFTP connection.</fsummary> <type> <v>SSH = connection()</v> @@ -369,7 +369,7 @@ </func> <func> - <name>exec(SSH, Command) -> {ok, Data} | {error, Reason}</name> + <name since="">exec(SSH, Command) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to exec(SSH, Command, DefaultTimeout).</fsummary> <desc><marker id="exec-2"/> <p>Equivalent to @@ -379,7 +379,7 @@ </func> <func> - <name>exec(SSH, Command, Timeout) -> {ok, Data} | {error, Reason}</name> + <name since="">exec(SSH, Command, Timeout) -> {ok, Data} | {error, Reason}</name> <fsummary>Requests server to perform Command.</fsummary> <type> <v>SSH = connection()</v> @@ -396,7 +396,7 @@ </func> <func> - <name>exec(SSH, ChannelId, Command, Timeout) -> {ok, Data} | {error, Reason}</name> + <name since="">exec(SSH, ChannelId, Command, Timeout) -> {ok, Data} | {error, Reason}</name> <fsummary>Requests server to perform Command.</fsummary> <type> <v>SSH = connection()</v> @@ -414,7 +414,7 @@ </func> <func> - <name>get_file_info(SSH, Handle) -> Result</name> + <name since="">get_file_info(SSH, Handle) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -428,7 +428,7 @@ </func> <func> - <name>get_file_info(SSH, Server, Handle) -> Result</name> + <name since="">get_file_info(SSH, Server, Handle) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -442,7 +442,7 @@ </func> <func> - <name>list_dir(SSH, Path) -> Result</name> + <name since="">list_dir(SSH, Path) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -456,7 +456,7 @@ </func> <func> - <name>list_dir(SSH, Server, Path) -> Result</name> + <name since="">list_dir(SSH, Server, Path) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -470,7 +470,7 @@ </func> <func> - <name>make_dir(SSH, Name) -> Result</name> + <name since="">make_dir(SSH, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -484,7 +484,7 @@ </func> <func> - <name>make_dir(SSH, Server, Name) -> Result</name> + <name since="">make_dir(SSH, Server, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -498,7 +498,7 @@ </func> <func> - <name>make_symlink(SSH, Name, Target) -> Result</name> + <name since="">make_symlink(SSH, Name, Target) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -512,7 +512,7 @@ </func> <func> - <name>make_symlink(SSH, Server, Name, Target) -> Result</name> + <name since="">make_symlink(SSH, Server, Name, Target) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -526,7 +526,7 @@ </func> <func> - <name>open(SSH, File, Mode) -> Result</name> + <name since="">open(SSH, File, Mode) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -540,7 +540,7 @@ </func> <func> - <name>open(SSH, Server, File, Mode) -> Result</name> + <name since="">open(SSH, Server, File, Mode) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -554,7 +554,7 @@ </func> <func> - <name>opendir(SSH, Path) -> Result</name> + <name since="">opendir(SSH, Path) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -568,7 +568,7 @@ </func> <func> - <name>opendir(SSH, Server, Path) -> Result</name> + <name since="">opendir(SSH, Server, Path) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -582,7 +582,7 @@ </func> <func> - <name>position(SSH, Handle, Location) -> Result</name> + <name since="">position(SSH, Handle, Location) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -596,7 +596,7 @@ </func> <func> - <name>position(SSH, Server, Handle, Location) -> Result</name> + <name since="">position(SSH, Server, Handle, Location) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -610,7 +610,7 @@ </func> <func> - <name>pread(SSH, Handle, Position, Length) -> Result</name> + <name since="">pread(SSH, Handle, Position, Length) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -624,7 +624,7 @@ </func> <func> - <name>pread(SSH, Server, Handle, Position, Length) -> Result</name> + <name since="">pread(SSH, Server, Handle, Position, Length) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -638,7 +638,7 @@ </func> <func> - <name>pwrite(SSH, Handle, Position, Data) -> Result</name> + <name since="">pwrite(SSH, Handle, Position, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -652,7 +652,7 @@ </func> <func> - <name>pwrite(SSH, Server, Handle, Position, Data) -> Result</name> + <name since="">pwrite(SSH, Server, Handle, Position, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -666,7 +666,7 @@ </func> <func> - <name>read(SSH, Handle, Len) -> Result</name> + <name since="">read(SSH, Handle, Len) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -680,7 +680,7 @@ </func> <func> - <name>read(SSH, Server, Handle, Len) -> Result</name> + <name since="">read(SSH, Server, Handle, Len) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -694,7 +694,7 @@ </func> <func> - <name>read_file(SSH, File) -> Result</name> + <name since="">read_file(SSH, File) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -708,7 +708,7 @@ </func> <func> - <name>read_file(SSH, Server, File) -> Result</name> + <name since="">read_file(SSH, Server, File) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -722,7 +722,7 @@ </func> <func> - <name>read_file_info(SSH, Name) -> Result</name> + <name since="">read_file_info(SSH, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -736,7 +736,7 @@ </func> <func> - <name>read_file_info(SSH, Server, Name) -> Result</name> + <name since="">read_file_info(SSH, Server, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -750,7 +750,7 @@ </func> <func> - <name>read_link(SSH, Name) -> Result</name> + <name since="">read_link(SSH, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -764,7 +764,7 @@ </func> <func> - <name>read_link(SSH, Server, Name) -> Result</name> + <name since="">read_link(SSH, Server, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -778,7 +778,7 @@ </func> <func> - <name>read_link_info(SSH, Name) -> Result</name> + <name since="">read_link_info(SSH, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -792,7 +792,7 @@ </func> <func> - <name>read_link_info(SSH, Server, Name) -> Result</name> + <name since="">read_link_info(SSH, Server, Name) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -806,7 +806,7 @@ </func> <func> - <name>receive_response(SSH, ChannelId) -> {ok, Data} | {error, Reason}</name> + <name since="">receive_response(SSH, ChannelId) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to receive_response(SSH, ChannelId, close).</fsummary> <desc><marker id="receive_response-2"/> @@ -817,7 +817,7 @@ ChannelId, close)</c></seealso>.</p> </func> <func> - <name>receive_response(SSH, ChannelId, End) -> {ok, Data} | {error, Reason}</name> + <name since="">receive_response(SSH, ChannelId, End) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to receive_response(SSH, ChannelId, End, DefaultTimeout).</fsummary> <desc><marker id="receive_response-3"/> @@ -828,7 +828,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>receive_response(SSH, ChannelId, End, Timeout) -> {ok, Data} | {timeout, Data} | {error, Reason}</name> + <name since="">receive_response(SSH, ChannelId, End, Timeout) -> {ok, Data} | {timeout, Data} | {error, Reason}</name> <fsummary>Receives expected data from server on the specified session channel.</fsummary> <type> @@ -863,7 +863,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>rename(SSH, OldName, NewName) -> Result</name> + <name since="">rename(SSH, OldName, NewName) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -877,7 +877,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>rename(SSH, Server, OldName, NewName) -> Result</name> + <name since="">rename(SSH, Server, OldName, NewName) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -891,7 +891,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>send(SSH, ChannelId, Data) -> ok | {error, Reason}</name> + <name since="">send(SSH, ChannelId, Data) -> ok | {error, Reason}</name> <fsummary>Equivalent to send(SSH, ChannelId, 0, Data, DefaultTimeout).</fsummary> <desc><marker id="send-3"/> @@ -901,7 +901,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>send(SSH, ChannelId, Data, Timeout) -> ok | {error, Reason}</name> + <name since="">send(SSH, ChannelId, Data, Timeout) -> ok | {error, Reason}</name> <fsummary>Equivalent to send(SSH, ChannelId, 0, Data, Timeout).</fsummary> <desc><marker id="send-4"/> <p>Equivalent to <seealso marker="#send-5"><c>ct_ssh:send(SSH, @@ -910,7 +910,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>send(SSH, ChannelId, Type, Data, Timeout) -> ok | {error, Reason}</name> + <name since="">send(SSH, ChannelId, Type, Data, Timeout) -> ok | {error, Reason}</name> <fsummary>Sends data to server on specified session channel.</fsummary> <type> <v>SSH = connection()</v> @@ -926,7 +926,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>send_and_receive(SSH, ChannelId, Data) -> {ok, Data} | {error, Reason}</name> + <name since="">send_and_receive(SSH, ChannelId, Data) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to send_and_receive(SSH, ChannelId, Data, close).</fsummary> <desc><marker id="send_and_receive-3"/> @@ -937,7 +937,7 @@ ChannelId, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>send_and_receive(SSH, ChannelId, Data, End) -> {ok, Data} | {error, Reason}</name> + <name since="">send_and_receive(SSH, ChannelId, Data, End) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to send_and_receive(SSH, ChannelId, 0, Data, End, DefaultTimeout).</fsummary> <desc><marker id="send_and_receive-4"/> @@ -948,7 +948,7 @@ ChannelId, 0, Data, End, DefaultTimeout)</c></seealso>.</p> </func> <func> - <name>send_and_receive(SSH, ChannelId, Data, End, Timeout) -> {ok, Data} | {error, Reason}</name> + <name since="">send_and_receive(SSH, ChannelId, Data, End, Timeout) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to send_and_receive(SSH, ChannelId, 0, Data, End, Timeout).</fsummary> <desc><marker id="send_and_receive-5"/> @@ -959,7 +959,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>send_and_receive(SSH, ChannelId, Type, Data, End, Timeout) -> {ok, Data} | {error, Reason}</name> + <name since="">send_and_receive(SSH, ChannelId, Type, Data, End, Timeout) -> {ok, Data} | {error, Reason}</name> <fsummary>Sends data to server on specified session channel and waits to receive the server response.</fsummary> <type> @@ -981,7 +981,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>session_close(SSH, ChannelId) -> ok | {error, Reason}</name> + <name since="">session_close(SSH, ChannelId) -> ok | {error, Reason}</name> <fsummary>Closes an SSH session channel.</fsummary> <type> <v>SSH = connection()</v> @@ -994,7 +994,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>session_open(SSH) -> {ok, ChannelId} | {error, Reason}</name> + <name since="">session_open(SSH) -> {ok, ChannelId} | {error, Reason}</name> <fsummary>Equivalent to session_open(SSH, DefaultTimeout).</fsummary> <desc><marker id="session_open-1"/> <p>Equivalent to @@ -1004,7 +1004,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>session_open(SSH, Timeout) -> {ok, ChannelId} | {error, Reason}</name> + <name since="">session_open(SSH, Timeout) -> {ok, ChannelId} | {error, Reason}</name> <fsummary>Opens a channel for an SSH session.</fsummary> <type> <v>SSH = connection()</v> @@ -1018,7 +1018,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>sftp_connect(SSH) -> {ok, Server} | {error, Reason}</name> + <name since="">sftp_connect(SSH) -> {ok, Server} | {error, Reason}</name> <fsummary>Starts an SFTP session on an already existing SSH connection.</fsummary> <type> @@ -1034,7 +1034,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>shell(SSH, ChannelId) -> ok | {error, Reason}</name> + <name since="OTP 20.0">shell(SSH, ChannelId) -> ok | {error, Reason}</name> <fsummary>Equivalent to shell(SSH, ChannelId, DefaultTimeout).</fsummary> <desc><marker id="shell-2"/> <p>Equivalent to @@ -1044,7 +1044,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>shell(SSH, ChannelId, Timeout) -> ok | {error, Reason}</name> + <name since="OTP 20.0">shell(SSH, ChannelId, Timeout) -> ok | {error, Reason}</name> <fsummary>Requests that the user default shell is executed at the server end.</fsummary> <type> @@ -1061,7 +1061,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>subsystem(SSH, ChannelId, Subsystem) -> Status | {error, Reason}</name> + <name since="">subsystem(SSH, ChannelId, Subsystem) -> Status | {error, Reason}</name> <fsummary>Equivalent to subsystem(SSH, ChannelId, Subsystem, DefaultTimeout).</fsummary> <desc><marker id="subsystem-3"/> @@ -1072,7 +1072,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>subsystem(SSH, ChannelId, Subsystem, Timeout) -> Status | {error, Reason}</name> + <name since="">subsystem(SSH, ChannelId, Subsystem, Timeout) -> Status | {error, Reason}</name> <fsummary>Sends a request to execute a predefined subsystem.</fsummary> <type> <v>SSH = connection()</v> @@ -1088,7 +1088,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>write(SSH, Handle, Data) -> Result</name> + <name since="">write(SSH, Handle, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -1102,7 +1102,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>write(SSH, Server, Handle, Data) -> Result</name> + <name since="">write(SSH, Server, Handle, Data) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -1116,7 +1116,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>write_file(SSH, File, Iolist) -> Result</name> + <name since="">write_file(SSH, File, Iolist) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -1130,7 +1130,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>write_file(SSH, Server, File, Iolist) -> Result</name> + <name since="">write_file(SSH, Server, File, Iolist) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -1144,7 +1144,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>write_file_info(SSH, Name, Info) -> Result</name> + <name since="">write_file_info(SSH, Name, Info) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> @@ -1158,7 +1158,7 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> - <name>write_file_info(SSH, Server, Name, Info) -> Result</name> + <name since="">write_file_info(SSH, Server, Name, Info) -> Result</name> <fsummary>For information and other types, see ssh_sftp(3).</fsummary> <type> <v>SSH = connection()</v> diff --git a/lib/common_test/doc/src/ct_telnet.xml b/lib/common_test/doc/src/ct_telnet.xml index 8e85cccc99..76f5305c46 100644 --- a/lib/common_test/doc/src/ct_telnet.xml +++ b/lib/common_test/doc/src/ct_telnet.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_telnet.xml</file> </header> - <module>ct_telnet</module> + <module since="">ct_telnet</module> <modulesummary>Common Test specific layer on top of Telnet client ct_telnet_client.erl</modulesummary> <description> @@ -205,7 +205,7 @@ <funcs> <func> - <name>close(Connection) -> ok | {error, Reason}</name> + <name since="">close(Connection) -> ok | {error, Reason}</name> <fsummary>Closes the Telnet connection and stops the process managing it.</fsummary> <type> @@ -223,7 +223,7 @@ </func> <func> - <name>cmd(Connection, Cmd) -> {ok, Data} | {error, Reason}</name> + <name since="">cmd(Connection, Cmd) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to cmd(Connection, Cmd, []).</fsummary> <desc><marker id="cmd-2"/> <p>Equivalent to @@ -233,24 +233,27 @@ </func> <func> - <name>cmd(Connection, Cmd, Opts) -> {ok, Data} | {error, Reason}</name> + <name since="">cmd(Connection, Cmd, Opts) -> {ok, Data} | {error, Reason}</name> <fsummary>Sends a command through Telnet and waits for prompt.</fsummary> <type> <v>Connection = connection()</v> <v>Cmd = string()</v> <v>Opts = [Opt]</v> - <v>Opt = {timeout, timeout()} | {newline, boolean()}</v> + <v>Opt = {timeout, timeout()} | {newline, boolean() | string()}</v> <v>Data = [string()]</v> <v>Reason = term()</v> </type> <desc><marker id="cmd-3"/> <p>Sends a command through Telnet and waits for prompt.</p> - <p>By default, this function adds a new line to the end of the + <p>By default, this function adds "\n" to the end of the specified command. If this is not desired, use option <c>{newline,false}</c>. This is necessary, for example, when sending Telnet command sequences prefixed with character - Interprete As Command (IAC).</p> + Interpret As Command (IAC). Option <c>{newline,string()}</c> + can also be used if a different line end than "\n" is + required, for instance <c>{newline,"\r\n"}</c>, to add both + carriage return and newline characters.</p> <p>Option <c>timeout</c> specifies how long the client must wait for prompt. If the time expires, the function returns @@ -262,7 +265,7 @@ </func> <func> - <name>cmdf(Connection, CmdFormat, Args) -> {ok, Data} | {error, Reason}</name> + <name since="">cmdf(Connection, CmdFormat, Args) -> {ok, Data} | {error, Reason}</name> <fsummary>Equivalent to cmdf(Connection, CmdFormat, Args, []).</fsummary> <desc><marker id="cmdf-3"/> <p>Equivalent to @@ -272,7 +275,7 @@ </func> <func> - <name>cmdf(Connection, CmdFormat, Args, Opts) -> {ok, Data} | {error, Reason}</name> + <name since="">cmdf(Connection, CmdFormat, Args, Opts) -> {ok, Data} | {error, Reason}</name> <fsummary>Sends a Telnet command and waits for prompt (uses a format string and a list of arguments to build the command).</fsummary> <type> @@ -280,7 +283,7 @@ <v>CmdFormat = string()</v> <v>Args = list()</v> <v>Opts = [Opt]</v> - <v>Opt = {timeout, timeout()} | {newline, boolean()}</v> + <v>Opt = {timeout, timeout()} | {newline, boolean() | string()}</v> <v>Data = [string()]</v> <v>Reason = term()</v> </type> @@ -294,7 +297,7 @@ </func> <func> - <name>expect(Connection, Patterns) -> term()</name> + <name since="">expect(Connection, Patterns) -> term()</name> <fsummary>Equivalent to expect(Connections, Patterns, []).</fsummary> <desc><marker id="expect-2"/> <p>Equivalent to @@ -304,7 +307,7 @@ </func> <func> - <name>expect(Connection, Patterns, Opts) -> {ok, Match} | {ok, MatchList, HaltReason} | {error, Reason}</name> + <name since="">expect(Connection, Patterns, Opts) -> {ok, Match} | {ok, MatchList, HaltReason} | {error, Reason}</name> <fsummary>Gets data from Telnet and waits for the expected pattern.</fsummary> <type> @@ -339,7 +342,7 @@ subexpression number <c>N</c>. Subexpressions are denoted with <c>'(' ')'</c> in the regular expression.</p> - <p>If a <c>Tag</c> is speciifed, the returned <c>Match</c> also + <p>If a <c>Tag</c> is specified, the returned <c>Match</c> also includes the matched <c>Tag</c>. Otherwise, only <c>RxMatch</c> is returned.</p> @@ -382,7 +385,7 @@ can abort the operation of waiting for prompt.</p></item> <tag><c>repeat | repeat, N</c></tag> <item><p>The pattern(s) must be matched multiple times. If <c>N</c> - is speciified, the pattern(s) are matched <c>N</c> times, and + is specified, the pattern(s) are matched <c>N</c> times, and the function returns <c>HaltReason = done</c>. This option can be interrupted by one or more <c>HaltPatterns</c>. <c>MatchList</c> is always returned, that is, a list of <c>Match</c> instead of @@ -422,7 +425,7 @@ </func> <func> - <name>get_data(Connection) -> {ok, Data} | {error, Reason}</name> + <name since="">get_data(Connection) -> {ok, Data} | {error, Reason}</name> <fsummary>Gets all data received by the Telnet client since the last command was sent.</fsummary> <type> @@ -446,7 +449,7 @@ </func> <func> - <name>open(Name) -> {ok, Handle} | {error, Reason}</name> + <name since="">open(Name) -> {ok, Handle} | {error, Reason}</name> <fsummary>Equivalent to open(Name, telnet).</fsummary> <desc><marker id="open-1"/> <p>Equivalent to @@ -456,7 +459,7 @@ </func> <func> - <name>open(Name, ConnType) -> {ok, Handle} | {error, Reason}</name> + <name since="">open(Name, ConnType) -> {ok, Handle} | {error, Reason}</name> <fsummary>Opens a Telnet connection to the specified target host.</fsummary> <type> @@ -471,7 +474,7 @@ </func> <func> - <name>open(KeyOrName, ConnType, TargetMod) -> {ok, Handle} | {error, Reason}</name> + <name since="">open(KeyOrName, ConnType, TargetMod) -> {ok, Handle} | {error, Reason}</name> <fsummary>Equivalent to open(KeyOrName, ConnType, TargetMod, []).</fsummary> <desc><marker id="open-3"/> <p>Equivalent to @@ -481,7 +484,7 @@ </func> <func> - <name>open(KeyOrName, ConnType, TargetMod, Extra) -> {ok, Handle} | {error, Reason}</name> + <name since="">open(KeyOrName, ConnType, TargetMod, Extra) -> {ok, Handle} | {error, Reason}</name> <fsummary>Opens a Telnet connection to the specified target host.</fsummary> <type> @@ -531,7 +534,7 @@ </func> <func> - <name>send(Connection, Cmd) -> ok | {error, Reason}</name> + <name since="">send(Connection, Cmd) -> ok | {error, Reason}</name> <fsummary>Equivalent to send(Connection, Cmd, []).</fsummary> <desc><marker id="send-2"/> <p>Equivalent to @@ -541,23 +544,26 @@ </func> <func> - <name>send(Connection, Cmd, Opts) -> ok | {error, Reason}</name> + <name since="OTP 17.4">send(Connection, Cmd, Opts) -> ok | {error, Reason}</name> <fsummary>Sends a Telnet command and returns immediately.</fsummary> <type> <v>Connection = connection()</v> <v>Cmd = string()</v> <v>Opts = [Opt]</v> - <v>Opt = {newline, boolean()}</v> + <v>Opt = {newline, boolean() | string()}</v> <v>Reason = term()</v> </type> <desc><marker id="send-3"/> <p>Sends a Telnet command and returns immediately.</p> - <p>By default, this function adds a newline to the end of the + <p>By default, this function adds "\n" to the end of the specified command. If this is not desired, option <c>{newline,false}</c> can be used. This is necessary, for example, when sending Telnet command sequences prefixed with character - Interprete As Command (IAC).</p> + Interpret As Command (IAC). Option <c>{newline,string()}</c> + can also be used if a different line end than "\n" is + required, for instance <c>{newline,"\r\n"}</c>, to add both + carriage return and newline characters.</p> <p>The resulting output from the command can be read with <seealso marker="#get_data-1"><c>ct_telnet:get_data/2</c></seealso> or @@ -566,7 +572,7 @@ </func> <func> - <name>sendf(Connection, CmdFormat, Args) -> ok | {error, Reason}</name> + <name since="">sendf(Connection, CmdFormat, Args) -> ok | {error, Reason}</name> <fsummary>Equivalent to sendf(Connection, CmdFormat, Args, []).</fsummary> <desc><marker id="sendf-3"/> <p>Equivalent to @@ -576,7 +582,7 @@ </func> <func> - <name>sendf(Connection, CmdFormat, Args, Opts) -> ok | {error, Reason}</name> + <name since="OTP 17.4">sendf(Connection, CmdFormat, Args, Opts) -> ok | {error, Reason}</name> <fsummary>Sends a Telnet command and returns immediately (uses a format string and a list of arguments to build the command).</fsummary> <type> @@ -584,12 +590,15 @@ <v>CmdFormat = string()</v> <v>Args = list()</v> <v>Opts = [Opt]</v> - <v>Opt = {newline, boolean()}</v> + <v>Opt = {newline, boolean() | string()}</v> <v>Reason = term()</v> </type> <desc><marker id="sendf-4"/> <p>Sends a Telnet command and returns immediately (uses a format string and a list of arguments to build the command).</p> + + <p>For details, see + <seealso marker="#send-3"><c>ct_telnet:send/3</c></seealso>.</p> </desc> </func> </funcs> diff --git a/lib/common_test/doc/src/ct_testspec.xml b/lib/common_test/doc/src/ct_testspec.xml index 36893f66cf..9cb9a2ae9f 100644 --- a/lib/common_test/doc/src/ct_testspec.xml +++ b/lib/common_test/doc/src/ct_testspec.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>ct_testspec.xml</file> </header> - <module>ct_testspec</module> + <module since="OTP 19.3">ct_testspec</module> <modulesummary>Parsing of test specifications for Common Test. </modulesummary> @@ -46,7 +46,7 @@ <funcs> <func> - <name>get_tests(SpecsIn) -> {ok, [{Specs,Tests}]} | {error, Reason}</name> + <name since="OTP 19.3">get_tests(SpecsIn) -> {ok, [{Specs,Tests}]} | {error, Reason}</name> <fsummary>Parse the given test specification files and return the tests to run and skip.</fsummary> <type> <v>SpecsIn = [string()] | [[string()]]</v> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 118dcd88bd..d39e86deb3 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -33,6 +33,81 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.17.1</title> + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + OTP internal test improvements.</p> + <p> + Own Id: OTP-15716</p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.17</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A bug caused <c>ct:encrypt_config_file/3</c> and + <c>ct:decrypt_config_file/3</c> to fail with + <c>badmatch</c> if input parameter <c>KeyOrFile</c> was + <c>{key,string()}</c>. This is now corrected.</p> + <p> + Own Id: OTP-15540</p> + </item> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A new variant of the <c>newline</c> option to + <c>ct_telnet:cmd/3</c> and <c>ct_telnet:send/3</c> is + added, which allows to specify a string to append as + newline indicator on a command. By default, the value is + "\n", but in some cases it is required to be "\r\n", + which this option allows.</p> + <p> + A faulty regular expression given as parameter to + <c>ct_telnet:expect/2,3</c> would earlier crash and look + like an internal error in common_test. A better error + indication is now given, but the test case will still + fail.</p> + <p> + Own Id: OTP-15229 Aux Id: ERIERL-203 </p> + </item> + <item> + <p> + Since the yang RFC allows more than one top element of + config data in an <c>edit-config</c> element, + <c>ct_netconfc:edit_config/3,4,5</c> can now take a list + of XML elements.</p> + <p> + Own Id: OTP-15298</p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.16.1</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -75,6 +150,110 @@ </section> +<section><title>Common_Test 1.15.4.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The test result when a hook function fails is in general + the same as if the function that the hook is associated + with fails. For example, if <c>post_init_per_testcase</c> + fails the result is that the test case is skipped, as is + the case when <c>init_per_testcase</c> fails.This, + however, was earlier not true for timetrap timeouts or + other error situations where the process running the hook + function was killed. This is now corrected, so the error + handling should be the same no matter how the hook + function fails.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15717 Aux Id: ERIERL-334 </p> + </item> + <item> + <p> + In some rare cases, when two common_test nodes used the + same log directory, a timing problem could occur which + caused common_test to crash because it's log cache file + was unexpectedly empty. This is now corrected.</p> + <p> + Own Id: OTP-15758 Aux Id: ERIERL-342 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Two new common_test hook functions are introduced:</p> + <p> + <c>post_groups/2</c>, which is called after + <c>Suite:groups/0</c><br/> <c>post_all/3</c>, which is + called after <c>Suite:all/0</c></p> + <p> + These functions allow modifying the return values from + the <c>groups/0</c> and <c>all/0</c> functions, + respectively.</p> + <p> + A new term, <c>{testcase,TestCase,RepeatProperties}</c> + is now also allowed in the return from <c>all/0</c>. This + can be used for repeating a single test case a specific + number of times, or until it fails or succeeds once.</p> + <p> + Own Id: OTP-14746 Aux Id: ERIERL-143 </p> + </item> + <item> + <p> + OTP internal test improvements.</p> + <p> + Own Id: OTP-15716</p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.15.4.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.15.4.0.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The status of a test case which failed with timetrap + timeout in <c>end_per_testcase</c> could not be modified + by returning <c>{fail,Reason}</c> from a + <c>post_end_per_testcase</c> hook function. This is now + corrected.</p> + <p> + Own Id: OTP-15584 Aux Id: ERIERL-282 </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.15.4</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -4026,8 +4205,3 @@ <section><title>common_test 1.3.0</title> </section> </chapter> - - - - - diff --git a/lib/common_test/doc/src/unix_telnet.xml b/lib/common_test/doc/src/unix_telnet.xml index b2314a53ec..03d91b7dbe 100644 --- a/lib/common_test/doc/src/unix_telnet.xml +++ b/lib/common_test/doc/src/unix_telnet.xml @@ -32,7 +32,7 @@ <rev>A</rev> <file>unix_telnet.xml</file> </header> - <module>unix_telnet</module> + <module since="">unix_telnet</module> <modulesummary>Callback module for ct_telnet, for connecting to a Telnet server on a UNIX host.</modulesummary> @@ -80,7 +80,7 @@ <funcs> <func> - <name>connect(ConnName, Ip, Port, Timeout, KeepAlive, TCPNoDelay, Extra) -> {ok, Handle} | {error, Reason}</name> + <name since="OTP 18.3.3">connect(ConnName, Ip, Port, Timeout, KeepAlive, TCPNoDelay, Extra) -> {ok, Handle} | {error, Reason}</name> <fsummary>Callback for ct_telnet.erl.</fsummary> <type> <v>ConnName = target_name()</v> @@ -107,7 +107,7 @@ </func> <func> - <name>get_prompt_regexp() -> PromptRegexp</name> + <name since="">get_prompt_regexp() -> PromptRegexp</name> <fsummary>Callback for ct_telnet.erl.</fsummary> <type> <v>PromptRegexp = prompt_regexp()</v> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 82dc06834f..5eed748b08 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -455,8 +455,10 @@ GroupDefs = [GroupDef] GroupDef = {GroupName,Properties,GroupsAndTestCases} GroupName = atom() - GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase] - TestCase = atom()</pre> + GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase | + {testcase,TestCase,TCRepeatProps}] + TestCase = atom() + TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</pre> <p><c>GroupName</c> is the name of the group and must be unique within the test suite module. Groups can be nested, by including a group definition @@ -464,11 +466,11 @@ <c>Properties</c> is the list of execution properties for the group. The possible values are as follows:</p> <pre> - Properties = [parallel | sequence | Shuffle | {RepeatType,N}] + Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] Shuffle = shuffle | {shuffle,Seed} Seed = {integer(),integer(),integer()} - RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | - repeat_until_any_ok | repeat_until_any_fail + GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | + repeat_until_any_ok | repeat_until_any_fail N = integer() | forever</pre> <p><em>Explanations:</em></p> @@ -481,8 +483,8 @@ Dependencies Between Test Cases and Suites.</p></item> <tag><c>shuffle</c></tag> <item><p>The cases in the group are executed in random order.</p></item> - <tag><c>repeat</c></tag> - <item><p>Orders <c>Common Test</c> to repeat execution of the cases in the + <tag><c>repeat, repeat_until_*</c></tag> + <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the group a given number of times, or until any, or all, cases fail or succeed.</p></item> </taglist> @@ -496,7 +498,7 @@ <c>{group,GroupName}</c> to the <c>all/0</c> list.</p> <p><em>Example:</em></p> <pre> - all() -> [testcase1, {group,group1}, testcase2, {group,group2}].</pre> + all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].</pre> <p>Execution properties with a group tuple in <c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified. diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index a10d939919..a07e61199b 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -592,7 +592,7 @@ encrypt_config_file(SrcFileName, EncryptFileName, {file,KeyFile}) -> encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) -> _ = crypto:start(), - {Key,IVec} = make_crypto_key(Key), + {CryptoKey,IVec} = make_crypto_key(Key), case file:read_file(SrcFileName) of {ok,Bin0} -> Bin1 = term_to_binary({SrcFileName,Bin0}), @@ -600,7 +600,7 @@ encrypt_config_file(SrcFileName, EncryptFileName, {key,Key}) -> 0 -> Bin1; N -> list_to_binary([Bin1,random_bytes(8-N)]) end, - EncBin = crypto:block_encrypt(des3_cbc, Key, IVec, Bin2), + EncBin = crypto:block_encrypt(des3_cbc, CryptoKey, IVec, Bin2), case file:write_file(EncryptFileName, EncBin) of ok -> io:format("~ts --(encrypt)--> ~ts~n", @@ -631,10 +631,10 @@ decrypt_config_file(EncryptFileName, TargetFileName, {file,KeyFile}) -> decrypt_config_file(EncryptFileName, TargetFileName, {key,Key}) -> _ = crypto:start(), - {Key,IVec} = make_crypto_key(Key), + {CryptoKey,IVec} = make_crypto_key(Key), case file:read_file(EncryptFileName) of {ok,Bin} -> - DecBin = crypto:block_decrypt(des3_cbc, Key, IVec, Bin), + DecBin = crypto:block_decrypt(des3_cbc, CryptoKey, IVec, Bin), case catch binary_to_term(DecBin) of {'EXIT',_} -> {error,bad_file}; diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 134ae0e1cc..dcf7c97244 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -696,9 +696,16 @@ end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> end end; -end_tc(Mod,Func0,TCPid,Result,Args,Return) -> +end_tc(Mod,Func00,TCPid,Result,Args,Return) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), + {OnlyCleanup,Func0} = + case Func00 of + {cleanup,F0} -> + {true,F0}; + _ -> + {false,Func00} + end, {Func,FuncSpec,HookFunc} = case Func0 of {end_per_testcase_not_run,F} -> @@ -742,6 +749,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> case HookFunc of undefined -> {ok,Result}; + _ when OnlyCleanup -> + {ok,Result}; _ -> case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of '$ct_no_change' -> @@ -752,6 +761,8 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, FinalResult = case get('$test_server_framework_test') of + _ when OnlyCleanup -> + Result1; undefined -> %% send sync notification so that event handlers may print %% in the log file before it gets closed @@ -1056,21 +1067,40 @@ group_or_func(Func, _Config) -> %%% should be returned. get_suite(Mod, all) -> - case catch apply(Mod, groups, []) of - {'EXIT',_} -> - get_all(Mod, []); - GroupDefs when is_list(GroupDefs) -> - case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of - {error,_} = Error -> - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Error properly - [{?MODULE,error_in_suite,[[Error]]}]; - ConfTests -> - get_all(Mod, ConfTests) - end; - _ -> + case safe_apply_groups_0(Mod,{ok,[]}) of + {ok,GroupDefs} -> + try ct_groups:find_groups(Mod, all, all, GroupDefs) of + ConfTests when is_list(ConfTests) -> + get_all(Mod, ConfTests) + catch + throw:{error,Error} -> + [{?MODULE,error_in_suite,[[{error,Error}]]}]; + _:Error:S -> + [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + end; + {error,{bad_return,_Bad}} -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", - [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] + [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]; + {error,{bad_hook_return,Bad}} -> + E = "Bad return value from post_groups/2 hook function", + [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}]; + {error,{failed,ExitReason}} -> + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:groups/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(?def_gl, ErrStr, []), + %% save the error info so it doesn't get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"), + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {error,What} -> + [{?MODULE,error_in_suite,[[{error,What}]]}] end; %%!============================================================ @@ -1080,54 +1110,74 @@ get_suite(Mod, all) -> %% group get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> - Name = ?val(name, Props), - case catch apply(Mod, groups, []) of - {'EXIT',_} -> - [Group]; - GroupDefs when is_list(GroupDefs) -> - case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of - {error,_} = Error -> - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Error properly - [{?MODULE,error_in_suite,[[Error]]}]; - [] -> - []; - ConfTests -> - case lists:member(skipped, Props) of - true -> - %% a *subgroup* specified *only* as skipped (and not - %% as an explicit test) should not be returned, or - %% init/end functions for top groups will be executed - case catch ?val(name, element(2, hd(ConfTests))) of - Name -> % top group - ct_groups:delete_subs(ConfTests, ConfTests); - _ -> - [] - end; - false -> - ConfTests1 = ct_groups:delete_subs(ConfTests, - ConfTests), - case ?val(override, Props) of - undefined -> - ConfTests1; - [] -> - ConfTests1; - ORSpec -> - ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; - true -> ORSpec end, - ct_groups:search_and_override(ConfTests1, - ORSpec1, Mod) - end - end - end; - _ -> + case safe_apply_groups_0(Mod,{ok,[Group]}) of + {ok,GroupDefs} -> + Name = ?val(name, Props), + try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of + [] -> + []; + ConfTests when is_list(ConfTests) -> + case lists:member(skipped, Props) of + true -> + %% a *subgroup* specified *only* as skipped (and not + %% as an explicit test) should not be returned, or + %% init/end functions for top groups will be executed + try ?val(name, element(2, hd(ConfTests))) of + Name -> % top group + ct_groups:delete_subs(ConfTests, ConfTests); + _ -> [] + catch + _:_ -> [] + end; + false -> + ConfTests1 = ct_groups:delete_subs(ConfTests, + ConfTests), + case ?val(override, Props) of + undefined -> + ConfTests1; + [] -> + ConfTests1; + ORSpec -> + ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; + true -> ORSpec end, + ct_groups:search_and_override(ConfTests1, + ORSpec1, Mod) + end + end + catch + throw:{error,Error} -> + [{?MODULE,error_in_suite,[[{error,Error}]]}]; + _:Error:S -> + [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + end; + {error,{bad_return,_Bad}} -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", - [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] + [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]; + {error,{bad_hook_return,Bad}} -> + E = "Bad return value from post_groups/2 hook function", + [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}]; + {error,{failed,ExitReason}} -> + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:groups/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(?def_gl, ErrStr, []), + %% save the error info so it doesn't get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"), + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {error,What} -> + [{?MODULE,error_in_suite,[[{error,What}]]}] end; %% testcase get_suite(Mod, Name) -> - get_seq(Mod, Name). + get_seq(Mod, Name). %%%----------------------------------------------------------------- @@ -1161,21 +1211,48 @@ get_all_cases1(_, []) -> %%%----------------------------------------------------------------- -get_all(Mod, ConfTests) -> - case catch apply(Mod, all, []) of - {'EXIT',{undef,[{Mod,all,[],_} | _]}} -> +get_all(Mod, ConfTests) -> + case safe_apply_all_0(Mod) of + {ok,AllTCs} -> + %% expand group references using ConfTests + try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of + {error,_} = Error -> + [{?MODULE,error_in_suite,[[Error]]}]; + Tests0 -> + Tests = ct_groups:delete_subs(Tests0, Tests0), + expand_tests(Mod, Tests) + catch + throw:{error,Error} -> + [{?MODULE,error_in_suite,[[{error,Error}]]}]; + _:Error:S -> + [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}] + end; + Skip = {skip,_Reason} -> + Skip; + {error,undef} -> + Reason = + case code:which(Mod) of + non_existing -> + list_to_atom( + atom_to_list(Mod)++ + " cannot be compiled or loaded"); + _ -> + list_to_atom( + atom_to_list(Mod)++":all/0 is missing") + end, + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Reason properly + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {error,{bad_return,_Bad}} -> Reason = - case code:which(Mod) of - non_existing -> - list_to_atom(atom_to_list(Mod)++ - " can not be compiled or loaded"); - _ -> - list_to_atom(atom_to_list(Mod)++":all/0 is missing") - end, - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Reason properly + list_to_atom("Bad return value from "++ + atom_to_list(Mod)++":all/0"), [{?MODULE,error_in_suite,[[{error,Reason}]]}]; - {'EXIT',ExitReason} -> + {error,{bad_hook_return,Bad}} -> + Reason = + list_to_atom("Bad return value from post_all/3 hook function"), + [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}]; + {error,{failed,ExitReason}} -> case ct_util:get_testdata({error_in_suite,Mod}) of undefined -> ErrStr = io_lib:format("~n*** ERROR *** " @@ -1192,28 +1269,8 @@ get_all(Mod, ConfTests) -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Reason properly [{?MODULE,error_in_suite,[[{error,Reason}]]}]; - AllTCs when is_list(AllTCs) -> - case catch save_seqs(Mod,AllTCs) of - {error,What} -> - [{?MODULE,error_in_suite,[[{error,What}]]}]; - SeqsAndTCs -> - %% expand group references in all() using ConfTests - case catch ct_groups:expand_groups(SeqsAndTCs, - ConfTests, - Mod) of - {error,_} = Error -> - [{?MODULE,error_in_suite,[[Error]]}]; - Tests -> - ct_groups:delete_subs(Tests, Tests) - end - end; - Skip = {skip,_Reason} -> - Skip; - _ -> - Reason = - list_to_atom("Bad return value from "++ - atom_to_list(Mod)++":all/0"), - [{?MODULE,error_in_suite,[[{error,Reason}]]}] + {error,What} -> + [{?MODULE,error_in_suite,[[{error,What}]]}] end. %%!============================================================ @@ -1571,3 +1628,74 @@ get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding) -> %%% -spec get_log_dir() -> {ok,LogDir} get_log_dir() -> ct_logs:get_log_dir(true). + +%%%----------------------------------------------------------------- +%%% Call all and group callbacks and post_* hooks with error handling +safe_apply_all_0(Mod) -> + try apply(Mod, all, []) of + AllTCs0 when is_list(AllTCs0) -> + try save_seqs(Mod,AllTCs0) of + SeqsAndTCs when is_list(SeqsAndTCs) -> + all_hook(Mod,SeqsAndTCs) + catch throw:{error,What} -> + {error,What} + end; + {skip,_}=Skip -> + all_hook(Mod,Skip); + Bad -> + {error,{bad_return,Bad}} + catch + _:Reason:Stacktrace -> + handle_callback_crash(Reason,Stacktrace,Mod,all,{error,undef}) + end. + +all_hook(Mod, All) -> + case ct_hooks:all(Mod, All) of + AllTCs when is_list(AllTCs) -> + {ok,AllTCs}; + {skip,_}=Skip -> + Skip; + {fail,Reason} -> + {error,Reason}; + Bad -> + {error,{bad_hook_return,Bad}} + end. + +safe_apply_groups_0(Mod,Default) -> + try apply(Mod, groups, []) of + GroupDefs when is_list(GroupDefs) -> + case ct_hooks:groups(Mod, GroupDefs) of + GroupDefs1 when is_list(GroupDefs1) -> + {ok,GroupDefs1}; + {fail,Reason} -> + {error,Reason}; + Bad -> + {error,{bad_hook_return,Bad}} + end; + Bad -> + {error,{bad_return,Bad}} + catch + _:Reason:Stacktrace -> + handle_callback_crash(Reason,Stacktrace,Mod,groups,Default) + end. + +handle_callback_crash(undef,[{Mod,Func,[],_}|_],Mod,Func,Default) -> + case ct_hooks:Func(Mod, []) of + [] -> + Default; + List when is_list(List) -> + {ok,List}; + {fail,Reason} -> + {error,Reason}; + Bad -> + {error,{bad_hook_return,Bad}} + end; +handle_callback_crash(Reason,Stacktrace,_Mod,_Func,_Default) -> + {error,{failed,{Reason,Stacktrace}}}. + +expand_tests(Mod, [{testcase,Case,[Prop]}|Tests]) -> + [{repeat,{Mod,Case},Prop}|expand_tests(Mod,Tests)]; +expand_tests(Mod,[Test|Tests]) -> + [Test|expand_tests(Mod,Tests)]; +expand_tests(_Mod,[]) -> + []. diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl index d867069dce..f4b12c41c0 100644 --- a/lib/common_test/src/ct_groups.erl +++ b/lib/common_test/src/ct_groups.erl @@ -101,23 +101,34 @@ find(Mod, [], TCs, Tests, _Known, _Defs, false) -> [{Mod,TC}]; ({group,_}) -> []; + ({testcase,TC,[Prop]}) when is_atom(TC), TC ==all -> + [{repeat,{Mod,TC},Prop}]; ({_,_}=TC) when TCs == all -> [TC]; - (TC) -> - if is_atom(TC) -> - Tuple = {Mod,TC}, - case lists:member(Tuple, TCs) of - true -> - [Tuple]; - false -> - case lists:member(TC, TCs) of - true -> [{Mod,TC}]; - false -> [] - end - end; - true -> - [] - end + (TC) when is_atom(TC) -> + Tuple = {Mod,TC}, + case lists:member(Tuple, TCs) of + true -> + [Tuple]; + false -> + case lists:member(TC, TCs) of + true -> [Tuple]; + false -> [] + end + end; + ({testcase,TC,[Prop]}) when is_atom(TC) -> + Tuple = {Mod,TC}, + case lists:member(Tuple, TCs) of + true -> + [{repeat,Tuple,Prop}]; + false -> + case lists:member(TC, TCs) of + true -> [{repeat,Tuple,Prop}]; + false -> [] + end + end; + (_) -> + [] end, Tests), if Cases == [] -> ['NOMATCH']; true -> Cases @@ -172,12 +183,19 @@ find(Mod, GrNames, all, [{M,TC} | Gs], Known, Defs, FindAll) when is_atom(M), M /= group, is_atom(TC) -> [{M,TC} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)]; +%% Save test case +find(Mod, GrNames, all, [{testcase,TC,[Prop]} | Gs], Known, + Defs, FindAll) when is_atom(TC) -> + [{repeat,{Mod,TC},Prop} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)]; + %% Check if test case should be saved -find(Mod, GrNames, TCs, [TC | Gs], Known, - Defs, FindAll) when is_atom(TC) orelse - ((size(TC) == 2) and (element(1,TC) /= group)) -> +find(Mod, GrNames, TCs, [TC | Gs], Known, Defs, FindAll) + when is_atom(TC) orelse + ((size(TC) == 3) andalso (element(1,TC) == testcase)) orelse + ((size(TC) == 2) and (element(1,TC) /= group)) -> Case = - if is_atom(TC) -> + case TC of + _ when is_atom(TC) -> Tuple = {Mod,TC}, case lists:member(Tuple, TCs) of true -> @@ -188,7 +206,18 @@ find(Mod, GrNames, TCs, [TC | Gs], Known, false -> [] end end; - true -> + {testcase,TC0,[Prop]} when is_atom(TC0) -> + Tuple = {Mod,TC0}, + case lists:member(Tuple, TCs) of + true -> + {repeat,Tuple,Prop}; + false -> + case lists:member(TC0, TCs) of + true -> {repeat,{Mod,TC0},Prop}; + false -> [] + end + end; + _ -> case lists:member(TC, TCs) of true -> {Mod,TC}; false -> [] @@ -289,12 +318,22 @@ modify_tc_list(GrSpecTs, TSCs, []) -> modify_tc_list1(GrSpecTs, TSCs); modify_tc_list(GrSpecTs, _TSCs, _) -> - [Test || Test <- GrSpecTs, not is_atom(Test)]. + [Test || Test <- GrSpecTs, not is_atom(Test), element(1,Test)=/=testcase]. modify_tc_list1(GrSpecTs, TSCs) -> %% remove all cases in group tc list that should not be executed GrSpecTs1 = - lists:flatmap(fun(Test) when is_tuple(Test), + lists:flatmap(fun(Test={testcase,TC,_}) -> + case lists:keysearch(TC, 2, TSCs) of + {value,_} -> + [Test]; + _ -> + case lists:member(TC, TSCs) of + true -> [Test]; + false -> [] + end + end; + (Test) when is_tuple(Test), (size(Test) > 2) -> [Test]; (Test={group,_}) -> diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index 49587b3edd..97c349578f 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -22,6 +22,8 @@ %% API Exports -export([init/1]). +-export([groups/2]). +-export([all/2]). -export([init_tc/3]). -export([end_tc/5]). -export([terminate/1]). @@ -37,7 +39,8 @@ opts = [], prio = ctfirst }]). --record(ct_hook_config, {id, module, prio, scope, opts = [], state = []}). +-record(ct_hook_config, {id, module, prio, scope, opts = [], + state = [], groups = []}). %% ------------------------------------------------------------------------- %% API Functions @@ -49,6 +52,48 @@ init(Opts) -> call(get_builtin_hooks(Opts) ++ get_new_hooks(Opts, undefined), ok, init, []). +%% Call the post_groups/2 hook callback +groups(Mod, Groups) -> + Info = try proplists:get_value(ct_hooks, Mod:suite(), []) of + CTHooks when is_list(CTHooks) -> + [{?config_name,CTHooks}]; + CTHook when is_atom(CTHook) -> + [{?config_name,[CTHook]}] + catch _:_ -> + %% since this might be the first time Mod:suite() + %% is called, and it might just fail or return + %% something bad, we allow any failure here - it + %% will be catched later if there is something + %% really wrong. + [{?config_name,[]}] + end, + case call(fun call_generic/3, Info ++ [{'$ct_groups',Groups}], [post_groups, Mod]) of + [{'$ct_groups',NewGroups}] -> + NewGroups; + Other -> + Other + end. + +%% Call the post_all/3 hook callback +all(Mod, Tests) -> + Info = try proplists:get_value(ct_hooks, Mod:suite(), []) of + CTHooks when is_list(CTHooks) -> + [{?config_name,CTHooks}]; + CTHook when is_atom(CTHook) -> + [{?config_name,[CTHook]}] + catch _:_ -> + %% just allow any failure here - it will be catched + %% later if there is something really wrong. + [{?config_name,[]}] + end, + case call(fun call_generic/3, Info ++ [{'$ct_all',Tests}], [post_all, Mod]) of + [{'$ct_all',NewTests}] -> + NewTests; + Other -> + Other + end. + +%% Called after all suites are done. -spec terminate(Hooks :: term()) -> ok. terminate(Hooks) -> @@ -80,6 +125,7 @@ init_tc(Mod, init_per_suite, Config) -> [{?config_name,[]}] end, call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]); + init_tc(Mod, end_per_suite, Config) -> call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); init_tc(Mod, {init_per_group, GroupName, Properties}, Config) -> @@ -153,7 +199,7 @@ call_id(#ct_hook_config{ module = Mod, opts = Opts} = Hook, Config, Scope) -> {Config, Hook#ct_hook_config{ id = Id, scope = scope(Scope)}}. call_init(#ct_hook_config{ module = Mod, opts = Opts, id = Id, prio = P} = Hook, - Config,_Meta) -> + Config, _Meta) -> case Mod:init(Id, Opts) of {ok, NewState} when P =:= undefined -> {Config, Hook#ct_hook_config{ state = NewState, prio = 0 } }; @@ -184,6 +230,18 @@ call_generic(Hook, Value, Meta) -> call_generic_fallback(Hook, Value, Meta) -> do_call_generic(Hook, Value, Meta, true). +do_call_generic(#ct_hook_config{ module = Mod} = Hook, + [{'$ct_groups',Groups}], [post_groups | Args], Fallback) -> + NewGroups = catch_apply(Mod, post_groups, Args ++ [Groups], + Groups, Fallback), + {[{'$ct_groups',NewGroups}], Hook#ct_hook_config{ groups = NewGroups } }; + +do_call_generic(#ct_hook_config{ module = Mod, groups = Groups} = Hook, + [{'$ct_all',Tests}], [post_all | Args], Fallback) -> + NewTests = catch_apply(Mod, post_all, Args ++ [Tests, Groups], + Tests, Fallback), + {[{'$ct_all',NewTests}], Hook}; + do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, Value, [Function | Args], Fallback) -> {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], @@ -218,6 +276,12 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> Rest ++ [{NewId, call_init}]}; ExistingHook when is_tuple(ExistingHook) -> {Hooks, Rest}; + _ when hd(Meta)=:=post_groups; hd(Meta)=:=post_all -> + %% If CTH is started because of a call from + %% groups/2 or all/2, CTH:init/1 must not be + %% called (the suite scope should be used). + {Hooks ++ [NewHook], + Rest ++ [{NewId,NextFun}]}; _ -> {Hooks ++ [NewHook], Rest ++ [{NewId, call_init}, {NewId,NextFun}]} @@ -226,8 +290,8 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) -> catch Error:Reason:Trace -> ct_logs:log("Suite Hook","Failed to start a CTH: ~tp:~tp", [Error,{Reason,Trace}]), - call([], {fail,"Failed to start CTH" - ", see the CT Log for details"}, Meta, Hooks) + call([], {fail,"Failed to start CTH, " + "see the CT Log for details"}, Meta, Hooks) end; call([{HookId, call_init} | Rest], Config, Meta, Hooks) -> call([{HookId, fun call_init/3} | Rest], Config, Meta, Hooks); @@ -267,6 +331,10 @@ scope([pre_init_per_suite, SuiteName|_]) -> [post_end_per_suite, SuiteName]; scope([post_init_per_suite, SuiteName|_]) -> [post_end_per_suite, SuiteName]; +scope([post_groups, SuiteName|_]) -> + [post_groups, SuiteName]; +scope([post_all, SuiteName|_]) -> + [post_all, SuiteName]; scope(init) -> none. @@ -353,6 +421,7 @@ resort(Calls,Hooks,[F|_R]) when F == pre_end_per_testcase; F == pre_end_per_suite; F == post_end_per_suite -> lists:reverse(resort(Calls,Hooks)); + resort(Calls,Hooks,_Meta) -> resort(Calls,Hooks). diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 07a1693d5d..a11613bca2 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -149,7 +149,7 @@ close(Info, StartDir) -> ok; CacheBin -> %% save final version of the log cache to file - _ = file:write_file(?log_cache_name,CacheBin), + write_log_cache(CacheBin), put(ct_log_cache,undefined) end end, @@ -2022,7 +2022,7 @@ update_all_runs_in_cache(AllRunsData) -> %% read from file as long as this logger process is alive put(ct_log_cache,term_to_binary(LogCache)); _ -> - file:write_file(?log_cache_name,term_to_binary(LogCache)) + write_log_cache(term_to_binary(LogCache)) end; SavedLogCache -> update_all_runs_in_cache(AllRunsData,binary_to_term(SavedLogCache)) @@ -2036,7 +2036,7 @@ update_all_runs_in_cache(AllRunsData, LogCache) -> %% read from file as long as this logger process is alive put(ct_log_cache,term_to_binary(LogCache1)); _ -> - file:write_file(?log_cache_name,term_to_binary(LogCache1)) + write_log_cache(term_to_binary(LogCache1)) end. sort_all_runs(Dirs) -> @@ -2668,7 +2668,7 @@ update_tests_in_cache(TempData,LogCache=#log_cache{tests=Tests}) -> {_Pid,_Pid} -> put(ct_log_cache,CacheBin); _ -> - file:write_file(?log_cache_name,CacheBin) + write_log_cache(CacheBin) end. %% @@ -3400,3 +3400,9 @@ unexpected_io(Pid, _Category, _Importance, Content, CtLogFd, EscChars) -> Data = io_lib:format("~ts", [lists:foldl(IoFun, [], Content)]), test_server_io:print_unexpected(Data), ok. + +write_log_cache(LogCacheBin) when is_binary(LogCacheBin) -> + TmpFile = ?log_cache_name++".tmp", + _ = file:write_file(TmpFile,LogCacheBin), + _ = file:rename(TmpFile,?log_cache_name), + ok. diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 29188a648e..6a758c4ea3 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -583,7 +583,7 @@ get_config(Client, Source, Filter, Timeout) -> -spec edit_config(Client, Target, Config) -> Result when Client :: client(), Target :: netconf_db(), - Config :: simple_xml(), + Config :: simple_xml() | [simple_xml()], Result :: ok | {error,error_reason()}. edit_config(Client, Target, Config) -> edit_config(Client, Target, Config, ?DEFAULT_TIMEOUT). @@ -591,7 +591,7 @@ edit_config(Client, Target, Config) -> -spec edit_config(Client, Target, Config, OptParams) -> Result when Client :: client(), Target :: netconf_db(), - Config :: simple_xml(), + Config :: simple_xml() | [simple_xml()], OptParams :: [simple_xml()], Result :: ok | {error,error_reason()}; (Client, Target, Config, Timeout) -> Result when @@ -608,10 +608,12 @@ edit_config(Client, Target, Config, OptParams) when is_list(OptParams) -> -spec edit_config(Client, Target, Config, OptParams, Timeout) -> Result when Client :: client(), Target :: netconf_db(), - Config :: simple_xml(), + Config :: simple_xml() | [simple_xml()], OptParams :: [simple_xml()], Timeout :: timeout(), Result :: ok | {error,error_reason()}. +edit_config(Client, Target, Config, OptParams, Timeout) when not is_list(Config)-> + edit_config(Client, Target, [Config], OptParams, Timeout); edit_config(Client, Target, Config, OptParams, Timeout) -> call(Client, {send_rpc_op, edit_config, [Target,Config,OptParams], Timeout}). @@ -1113,7 +1115,7 @@ encode_rpc_operation(get,[Filter]) -> encode_rpc_operation(get_config,[Source,Filter]) -> {'get-config',[{source,[Source]}] ++ filter(Filter)}; encode_rpc_operation(edit_config,[Target,Config,OptParams]) -> - {'edit-config',[{target,[Target]}] ++ OptParams ++ [{config,[Config]}]}; + {'edit-config',[{target,[Target]}] ++ OptParams ++ [{config,Config}]}; encode_rpc_operation(delete_config,[Target]) -> {'delete-config',[{target,[Target]}]}; encode_rpc_operation(copy_config,[Target,Source]) -> diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index f9abecfd38..174008c790 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -194,6 +194,15 @@ send(Connection,Cmd,Opts) -> check_send_opts([{newline,Bool}|Opts]) when is_boolean(Bool) -> check_send_opts(Opts); +check_send_opts([{newline,String}|Opts]) when is_list(String) -> + case lists:all(fun(I) when is_integer(I), I>=0, I=<127 -> true; + (_) -> false + end, String) of + true -> + check_send_opts(Opts); + false -> + {error,{invalid_option,{newline,String}}} + end; check_send_opts([Invalid|_]) -> {error,{invalid_option,Invalid}}; check_send_opts([]) -> @@ -211,10 +220,16 @@ expect(Connection,Patterns) -> expect(Connection,Patterns,Opts) -> case get_handle(Connection) of - {ok,Pid} -> - call(Pid,{expect,Patterns,Opts}); - Error -> - Error + {ok,Pid} -> + case call(Pid,{expect,Patterns,Opts}) of + {error,Reason} when element(1,Reason)==bad_pattern -> + %% Faulty user input - should fail the test case + exit({Reason,{?MODULE,?FUNCTION_NAME,3}}); + Other -> + Other + end; + Error -> + Error end. %%%================================================================= @@ -674,60 +689,68 @@ silent_teln_expect(Name,Pid,Data,Pattern,Prx,Opts) -> %% 3b) Repeat (sequence): 2) is repeated either N times or until a %% halt condition is fulfilled. teln_expect(Name,Pid,Data,Pattern0,Prx,Opts) -> - HaltPatterns = + HaltPatterns0 = case get_ignore_prompt(Opts) of true -> get_haltpatterns(Opts); false -> [prompt | get_haltpatterns(Opts)] end, - - PromptCheck = get_prompt_check(Opts), - - {WaitForPrompt,Pattern1,Opts1} = wait_for_prompt(Pattern0,Opts), - - Seq = get_seq(Opts1), - Pattern2 = convert_pattern(Pattern1,Seq), - {IdleTimeout,TotalTimeout} = get_timeouts(Opts1), - - EO = #eo{teln_pid=Pid, - prx=Prx, - idle_timeout=IdleTimeout, - total_timeout=TotalTimeout, - seq=Seq, - haltpatterns=HaltPatterns, - prompt_check=PromptCheck}, + case convert_pattern(HaltPatterns0,false) of + {ok,HaltPatterns} -> + {WaitForPrompt,Pattern1,Opts1} = wait_for_prompt(Pattern0,Opts), + Seq = get_seq(Opts1), + case convert_pattern(Pattern1,Seq) of + {ok,Pattern2} -> + {IdleTimeout,TotalTimeout} = get_timeouts(Opts1), + PromptCheck = get_prompt_check(Opts1), + + EO = #eo{teln_pid=Pid, + prx=Prx, + idle_timeout=IdleTimeout, + total_timeout=TotalTimeout, + seq=Seq, + haltpatterns=HaltPatterns, + prompt_check=PromptCheck}, - case get_repeat(Opts1) of - false -> - case teln_expect1(Name,Pid,Data,Pattern2,[],EO) of - {ok,Matched,Rest} when WaitForPrompt -> - case lists:reverse(Matched) of - [{prompt,_},Matched1] -> - {ok,Matched1,Rest}; - [{prompt,_}|Matched1] -> - {ok,lists:reverse(Matched1),Rest} - end; - {ok,Matched,Rest} -> - {ok,Matched,Rest}; - {halt,Why,Rest} -> - {error,Why,Rest}; - {error,Reason} -> - {error,Reason} - end; - N -> - EO1 = EO#eo{repeat=N}, - repeat_expect(Name,Pid,Data,Pattern2,[],EO1) + case get_repeat(Opts1) of + false -> + case teln_expect1(Name,Pid,Data,Pattern2,[],EO) of + {ok,Matched,Rest} when WaitForPrompt -> + case lists:reverse(Matched) of + [{prompt,_},Matched1] -> + {ok,Matched1,Rest}; + [{prompt,_}|Matched1] -> + {ok,lists:reverse(Matched1),Rest} + end; + {ok,Matched,Rest} -> + {ok,Matched,Rest}; + {halt,Why,Rest} -> + {error,Why,Rest}; + {error,Reason} -> + {error,Reason} + end; + N -> + EO1 = EO#eo{repeat=N}, + repeat_expect(Name,Pid,Data,Pattern2,[],EO1) + end; + Error -> + Error + end; + Error -> + Error end. -convert_pattern(Pattern,Seq) - when is_list(Pattern) and not is_integer(hd(Pattern)) -> - case Seq of - true -> Pattern; - false -> rm_dupl(Pattern,[]) - end; +convert_pattern(Pattern0,Seq) + when Pattern0==[] orelse (is_list(Pattern0) and not is_integer(hd(Pattern0))) -> + Pattern = + case Seq of + true -> Pattern0; + false -> rm_dupl(Pattern0,[]) + end, + compile_pattern(Pattern,[]); convert_pattern(Pattern,_Seq) -> - [Pattern]. + compile_pattern([Pattern],[]). rm_dupl([P|Ps],Acc) -> case lists:member(P,Acc) of @@ -739,6 +762,25 @@ rm_dupl([P|Ps],Acc) -> rm_dupl([],Acc) -> lists:reverse(Acc). +compile_pattern([prompt|Patterns],Acc) -> + compile_pattern(Patterns,[prompt|Acc]); +compile_pattern([{prompt,_}=P|Patterns],Acc) -> + compile_pattern(Patterns,[P|Acc]); +compile_pattern([{Tag,Pattern}|Patterns],Acc) -> + try re:compile(Pattern,[unicode]) of + {ok,MP} -> compile_pattern(Patterns,[{Tag,MP}|Acc]); + {error,Error} -> {error,{bad_pattern,{Tag,Pattern},Error}} + catch error:badarg -> {error,{bad_pattern,{Tag,Pattern}}} + end; +compile_pattern([Pattern|Patterns],Acc) -> + try re:compile(Pattern,[unicode]) of + {ok,MP} -> compile_pattern(Patterns,[MP|Acc]); + {error,Error} -> {error,{bad_pattern,Pattern,Error}} + catch error:badarg -> {error,{bad_pattern,Pattern}} + end; +compile_pattern([],Acc) -> + {ok,lists:reverse(Acc)}. + get_timeouts(Opts) -> {case lists:keysearch(idle_timeout,1,Opts) of {value,{_,T}} -> @@ -772,7 +814,7 @@ get_seq(Opts) -> get_haltpatterns(Opts) -> case lists:keysearch(halt,1,Opts) of {value,{halt,HaltPatterns}} -> - convert_pattern(HaltPatterns,false); + HaltPatterns; false -> [] end. @@ -1068,7 +1110,7 @@ match_line(Name,Pid,Line,[{prompt,PromptType}|Patterns],FoundPrompt,Term, when PromptType=/=FoundPrompt -> match_line(Name,Pid,Line,Patterns,FoundPrompt,Term,EO,RetTag); match_line(Name,Pid,Line,[{Tag,Pattern}|Patterns],FoundPrompt,Term,EO,RetTag) -> - case re:run(Line,Pattern,[{capture,all,list},unicode]) of + case re:run(Line,Pattern,[{capture,all,list}]) of nomatch -> match_line(Name,Pid,Line,Patterns,FoundPrompt,Term,EO,RetTag); {match,Match} -> @@ -1076,7 +1118,7 @@ match_line(Name,Pid,Line,[{Tag,Pattern}|Patterns],FoundPrompt,Term,EO,RetTag) -> {RetTag,{Tag,Match}} end; match_line(Name,Pid,Line,[Pattern|Patterns],FoundPrompt,Term,EO,RetTag) -> - case re:run(Line,Pattern,[{capture,all,list},unicode]) of + case re:run(Line,Pattern,[{capture,all,list}]) of nomatch -> match_line(Name,Pid,Line,Patterns,FoundPrompt,Term,EO,RetTag); {match,Match} -> diff --git a/lib/common_test/src/ct_telnet_client.erl b/lib/common_test/src/ct_telnet_client.erl index 76e4b9ea70..007477c855 100644 --- a/lib/common_test/src/ct_telnet_client.erl +++ b/lib/common_test/src/ct_telnet_client.erl @@ -101,9 +101,11 @@ close(Pid) -> end. send_data(Pid, Data) -> - send_data(Pid, Data, true). + send_data(Pid, Data, "\n"). send_data(Pid, Data, true) -> - send_data(Pid, Data++"\n", false); + send_data(Pid, Data, "\n"); +send_data(Pid, Data, Newline) when is_list(Newline) -> + send_data(Pid, Data++Newline, false); send_data(Pid, Data, false) -> Pid ! {send_data, Data}, ok. diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index a896a0551b..756cd4d692 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -384,8 +384,8 @@ run_test_case_apply({CaseNum,Mod,Func,Args,Name,RunInit,TimetrapData}) -> {Result,DetFail,ProcBef,ProcAft}. -type tc_status() :: 'starting' | 'running' | 'init_per_testcase' | - 'end_per_testcase' | {'framework',atom(),atom()} | - 'tc'. + 'end_per_testcase' | {'framework',{atom(),atom(),list}} | + 'tc'. -record(st, { ref :: reference(), @@ -653,8 +653,8 @@ handle_tc_exit({testcase_aborted,{user_timetrap_error,_}=Msg,_}, St) -> #st{config=Config,mf={Mod,Func},pid=Pid} = St, spawn_fw_call(Mod, Func, Config, Pid, Msg, unknown, self()), St; -handle_tc_exit(Reason, #st{status={framework,FwMod,FwFunc}, - config=Config,pid=Pid}=St) -> +handle_tc_exit(Reason, #st{status={framework,{FwMod,FwFunc,_}=FwMFA}, + config=Config,mf={Mod,Func},pid=Pid}=St) -> R = case Reason of {timetrap_timeout,TVal,_} -> {timetrap,TVal}; @@ -666,7 +666,7 @@ handle_tc_exit(Reason, #st{status={framework,FwMod,FwFunc}, Other end, Error = {framework_error,R}, - spawn_fw_call(FwMod, FwFunc, Config, Pid, Error, unknown, self()), + spawn_fw_call(Mod, Func, Config, Pid, {Error,FwMFA}, unknown, self()), St; handle_tc_exit(Reason, #st{status=tc,config=Config0,mf={Mod,Func},pid=Pid}=St) when is_list(Config0) -> @@ -850,36 +850,68 @@ spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid, "WARNING: end_per_testcase failed!</font>", {died,W} end, - try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of - _ -> ok - catch - _:FwEndTCErr -> - exit({fw_notify_done,end_tc,FwEndTCErr}) - end, - FailLoc = proplists:get_value(tc_fail_loc, EndConf), + FailLoc0 = proplists:get_value(tc_fail_loc, EndConf), + {RetVal1,FailLoc} = + try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of + Why -> + {RetVal,FailLoc0}; + {failed,_} = R -> + {R,[{Mod,Func}]}; + R -> + {R,FailLoc0} + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, %% finished, report back (if end_per_testcase fails, a warning %% should be printed as part of the comment) SendTo ! {self(),fw_notify_done, - {Time,RetVal,FailLoc,[],Warn}} + {Time,RetVal1,FailLoc,[],Warn}} end, spawn_link(FwCall); -spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> +spawn_fw_call(Mod,Func,Conf,Pid,{{framework_error,FwError}, + {FwMod,FwFunc,[A1,A2|_]}=FwMFA},_,SendTo) -> FwCall = fun() -> ct_util:mark_process(), - test_server_sup:framework_call(report, [framework_error, - {{FwMod,FwFunc}, - FwError}]), + Time = + case FwError of + {timetrap,TVal} -> + TVal/1000; + _ -> + died + end, + {Ret,Loc,WarnOrError} = + cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,FwMFA), Comment = - lists:flatten( - io_lib:format("<font color=\"red\">" - "WARNING! ~w:~tw failed!</font>", - [FwMod,FwFunc])), + case WarnOrError of + warn -> + group_leader() ! + {printout,12, + "WARNING! ~w:~tw(~w,~tw,...) failed!\n" + " Reason: ~tp\n", + [FwMod,FwFunc,A1,A2,FwError]}, + lists:flatten( + io_lib:format("<font color=\"red\">" + "WARNING! ~w:~tw(~w,~tw,...) " + "failed!</font>", + [FwMod,FwFunc,A1,A2])); + error -> + group_leader() ! + {printout,12, + "Error! ~w:~tw(~w,~tw,...) failed!\n" + " Reason: ~tp\n", + [FwMod,FwFunc,A1,A2,FwError]}, + lists:flatten( + io_lib:format("<font color=\"red\">" + "ERROR! ~w:~tw(~w,~tw,...) " + "failed!</font>", + [FwMod,FwFunc,A1,A2])) + end, %% finished, report back SendTo ! {self(),fw_notify_done, - {died,{error,{FwMod,FwFunc,FwError}}, - {FwMod,FwFunc},[],Comment}} + {Time,Ret,Loc,[],Comment}} end, spawn_link(FwCall); @@ -902,17 +934,185 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> FwErrorNotifyErr}) end, Conf = [{tc_status,{failed,Error}}|CurrConf], - try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of - _ -> ok - catch - _:FwEndTCErr -> - exit({fw_notify_done,end_tc,FwEndTCErr}) - end, + {Time,RetVal,Loc1} = + try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of + Error -> + {died, Error, Loc}; + {failed,Reason} = NewReturn -> + fw_error_notify(Mod,Func1,Conf,Reason), + {died, NewReturn, [{Mod,Func}]}; + NewReturn -> + T = case Error of + {timetrap_timeout,TT} -> TT; + _ -> 0 + end, + {T, NewReturn, Loc} + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, %% finished, report back - SendTo ! {self(),fw_notify_done,{died,Error,Loc,[],undefined}} + SendTo ! {self(),fw_notify_done,{Time,RetVal,Loc1,[],undefined}} end, spawn_link(FwCall). +cleanup_after_fw_error(_Mod,_Func,Conf,Pid,FwError, + {FwMod,FwFunc=init_tc, + [Mod,{init_per_testcase,Func}=IPTC|_]}) -> + %% Failed during pre_init_per_testcase, the test must be skipped + Skip = {auto_skip,{failed,{FwMod,FwFunc,FwError}}}, + try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[Conf]}, FwError), + do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, + [Conf],{ok,[Conf]}), + do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, + {Pid,Skip,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {Skip,{FwMod,FwFunc},error}; +cleanup_after_fw_error(_Mod,_Func,Conf,Pid,FwError, + {FwMod,FwFunc=end_tc,[Mod,{init_per_testcase,Func}|_]}) -> + %% Failed during post_init_per_testcase, the test must be skipped + Skip = {auto_skip,{failed,{FwMod,FwFunc,FwError}}}, + try begin do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, + [Conf],{ok,[Conf]}), + do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, + {Pid,Skip,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {Skip,{FwMod,FwFunc},error}; +cleanup_after_fw_error(_Mod,_Func,Conf,Pid,FwError, + {FwMod,FwFunc=init_tc,[Mod,{end_per_testcase,Func}|_]}) -> + %% Failed during pre_end_per_testcase. Warn about it. + {RetVal,Loc} = + case {proplists:get_value(tc_status, Conf), + proplists:get_value(tc_fail_loc, Conf, unknown)} of + {undefined,_} -> + {{failed,{FwMod,FwFunc,FwError}},{FwMod,FwFunc}}; + {E = {failed,_Reason},unknown} -> + {E,[{Mod,Func}]}; + {Result,FailLoc} -> + {Result,FailLoc} + end, + try begin do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, + {Pid,RetVal,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {RetVal,Loc,warn}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError, + {FwMod,FwFunc=end_tc,[Mod,{end_per_testcase,Func}|_]}) -> + %% Failed during post_end_per_testcase. Warn about it. + {RetVal,Report,Loc} = + case {proplists:get_value(tc_status, Conf), + proplists:get_value(tc_fail_loc, Conf, unknown)} of + {undefined,_} -> + {{failed,{FwMod,FwFunc,FwError}}, + {{FwMod,FwError},FwError}, + {FwMod,FwFunc}}; + {E = {failed,_Reason},unknown} -> + {E,{Mod,Func,E},[{Mod,Func}]}; + {Result,FailLoc} -> + {Result,{Mod,Func,Result},FailLoc} + end, + try begin do_end_tc_call(Mod,{cleanup,{end_per_testcase_not_run,Func}}, + {Pid,RetVal,[Conf]}, FwError) end of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + test_server_sup:framework_call(report,[framework_error,Report]), + {RetVal,Loc,warn}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=init_tc,_}) + when Func =:= init_per_suite; Func =:=init_per_group -> + %% Failed during pre_init_per_suite or pre_init_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,Func,{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=end_tc,_}) + when Func =:= init_per_suite; Func =:=init_per_group -> + %% Failed during post_init_per_suite or post_init_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,{cleanup,Func},{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + ReportFunc = + case Func of + init_per_group -> + case proplists:get_value(tc_group_properties,Conf) of + undefined -> + {Func,unknown,[]}; + GProps -> + Name = proplists:get_value(name,GProps), + {Func,Name,proplists:delete(name,GProps)} + end; + _ -> + Func + end, + test_server_sup:framework_call(report,[framework_error, + {Mod,ReportFunc,RetVal}]), + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=init_tc,_}) + when Func =:= end_per_suite; Func =:=end_per_group -> + %% Failed during pre_end_per_suite or pre_end_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,Func,{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(Mod,Func,Conf,Pid,FwError,{FwMod,FwFunc=end_tc,_}) + when Func =:= end_per_suite; Func =:=end_per_group -> + %% Failed during post_end_per_suite or post_end_per_group + RetVal = {failed,{FwMod,FwFunc,FwError}}, + try do_end_tc_call(Mod,{cleanup,Func},{Pid,RetVal,[Conf]},FwError) of + _ -> ok + catch + _:FwEndTCErr -> + exit({fw_notify_done,end_tc,FwEndTCErr}) + end, + ReportFunc = + case Func of + end_per_group -> + case proplists:get_value(tc_group_properties,Conf) of + undefined -> + {Func,unknown,[]}; + GProps -> + Name = proplists:get_value(name,GProps), + {Func,Name,proplists:delete(name,GProps)} + end; + _ -> + Func + end, + test_server_sup:framework_call(report,[framework_error, + {Mod,ReportFunc,RetVal}]), + {RetVal,{FwMod,FwFunc},error}; +cleanup_after_fw_error(_Mod,_Func,_Conf,_Pid,FwError,{FwMod,FwFunc,_}) -> + %% This is unexpected + test_server_sup:framework_call(report, + [framework_error, + {{FwMod,FwFunc}, + FwError}]), + {FwError,{FwMod,FwFunc},error}. + %% The job proxy process forwards messages between the test case %% process on a shielded node (and its descendants) and the job process. %% @@ -1088,6 +1288,9 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> EndConf1 = user_callback(TCCallback, Mod, Func, 'end', EndConf), + %% save updated config in controller loop + set_tc_state(tc, EndConf1), + %% We can't handle fails or skips here EndConf2 = case do_init_tc_call(Mod,{end_per_testcase,Func}, diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index 8bd6cd583a..fdcaaab2eb 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -1443,6 +1443,8 @@ remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) -> true -> remove_conf(Cases, [C|NoConf], Repeats) end; +remove_conf([C={repeat,_,_}|Cases], NoConf, _Repeats) -> + remove_conf(Cases, [C|NoConf], true); remove_conf([C|Cases], NoConf, Repeats) -> remove_conf(Cases, [C|NoConf], Repeats); remove_conf([], NoConf, true) -> @@ -2061,6 +2063,14 @@ add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod) [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; +add_init_and_end_per_suite([{repeat,{Mod,_},_}=Case|Cases], LastMod, LastRef, FwMod) + when Mod =/= LastMod, Mod =/= FwMod -> + {PreCases, NextMod, NextRef} = + do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([{repeat,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> + [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = @@ -2138,7 +2148,7 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> %% let's call a "fake" end_per_suite if it exists case erlang:function_exported(FwMod, end_per_suite, 1) of true -> - [{conf,LastRef,[{suite,Mod}], + [{conf,LastRef,[{suite,LastMod}], {FwMod,end_per_suite}}|Init]; false -> [{conf,LastRef,[],{LastMod,end_per_suite}}|Init] @@ -2926,6 +2936,29 @@ run_test_cases_loop([{conf,_Ref,_Props,_X}=Conf|_Cases0], Config, _TimetrapData, _Mode, _Status) -> erlang:error(badarg, [Conf,Config]); +run_test_cases_loop([{repeat,Case,{RepeatType,N}}|Cases0], Config, + TimeTrapData, Mode, Status) -> + Ref = make_ref(), + Parallel = check_prop(parallel, Mode) =/= false, + Sequence = check_prop(sequence, Mode) =/= false, + RepeatStop = RepeatType=:=repeat_until_fail + orelse RepeatType=:=repeat_until_ok, + + if Parallel andalso RepeatStop -> + %% Cannot check results of test case during parallal + %% execution, so only RepeatType=:=repeat is allowed in + %% combination with parallel groups. + erlang:error({illegal_combination,{parallel,RepeatType}}); + Sequence andalso RepeatStop -> + %% Sequence is stop on fail + skip rest, so only + %% RepeatType=:=repeat makes sense inside a sequence. + erlang:error({illegal_combination,{sequence,RepeatType}}); + true -> + Mode1 = [{Ref,[{repeat,{RepeatType,1,N}}],?now}|Mode], + run_test_cases_loop([Case | Cases0], Config, TimeTrapData, + Mode1, Status) + end; + run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) -> ActualCfg = case get(test_server_create_priv_dir) of @@ -2938,7 +2971,7 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) -> run_test_cases_loop([{Mod,Case,[ActualCfg]}|Cases], Config, TimetrapData, Mode, Status); -run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) -> +run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, Status) -> {Num,RunInit} = case FwMod = get_fw_mod(?MODULE) of Mod when Func == error_in_suite -> @@ -2948,6 +2981,14 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) run_init} end, + Mode = + case Mode0 of + [{_,[{repeat,{_,_,_}}],_}|RestMode] -> + RestMode; + _ -> + Mode0 + end, + %% check the current execution mode and save info about the case if %% detected that printouts to common log files is handled later @@ -2975,36 +3016,42 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) if is_tuple(RetVal) -> element(1,RetVal); true -> undefined end, - {Failed,Status1} = + {Result,Failed,Status1} = case RetTag of Skip when Skip==skip; Skip==skipped -> - {false,update_status(skipped, Mod, Func, Status)}; + {skipped,false,update_status(skipped, Mod, Func, Status)}; Fail when Fail=='EXIT'; Fail==failed -> - {true,update_status(failed, Mod, Func, Status)}; + {failed,true,update_status(failed, Mod, Func, Status)}; _ when Time==died, RetVal=/=ok -> - {true,update_status(failed, Mod, Func, Status)}; + {failed,true,update_status(failed, Mod, Func, Status)}; _ -> - {false,update_status(ok, Mod, Func, Status)} + {ok,false,update_status(ok, Mod, Func, Status)} end, case check_prop(sequence, Mode) of false -> + {Cases1,Mode1} = + check_repeat_testcase(Case,Result,Cases,Mode0), stop_minor_log_file(), - run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1); + run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1); Ref -> %% the case is in a sequence; we must check the result and %% determine if the following cases should run or be skipped if not Failed -> % proceed with next case + {Cases1,Mode1} = + check_repeat_testcase(Case,Result,Cases,Mode0), stop_minor_log_file(), - run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1); + run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1); true -> % skip rest of cases in sequence print(minor, "~n*** ~tw failed.~n" " Skipping all other cases in sequence.", [Func]), + {Cases1,Mode1} = + check_repeat_testcase(Case,Result,Cases,Mode0), Reason = {failed,{Mod,Func}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, + Cases2 = skip_cases_upto(Ref, Cases1, Reason, tc, Mode, auto_skip_case), stop_minor_log_file(), - run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1) + run_test_cases_loop(Cases2, Config, TimetrapData, Mode1, Status1) end end; %% the test case is being executed in parallel with the main process (and @@ -3013,7 +3060,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) %% io from Pid will be buffered by the test_server_io process and %% handled later, so we have to save info about the case queue_test_case_io(undefined, Pid, Num+1, Mod, Func), - run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status) + {Cases1,Mode1} = check_repeat_testcase(Case,ok,Cases,Mode0), + run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status) end; %% TestSpec processing finished @@ -3451,9 +3499,19 @@ modify_cases_upto1(Ref, {skip,Reason,FType,Mode,SkipType}, T, Orig, Alt) end; -%% next is some other case, ignore or copy -modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_Other|T], Orig, Alt) -> +%% next is a repeated test case +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op, + [{repeat,{_M,_F}=MF,_Repeat}|T], Orig, Alt) -> + modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{MF,Reason},Mode}|Alt]); + +%% next is an already skipped case, ignore or copy +modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [{SkipType,_,_}|T], Orig, Alt) + when SkipType=:=skip_case; SkipType=:=auto_skip_case -> modify_cases_upto1(Ref, Op, T, Orig, Alt); + +%% next is some other case, mark as skipped or copy +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op, [Other|T], Orig, Alt) -> + modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{Other,Reason},Mode}|Alt]); modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]). @@ -3841,6 +3899,10 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,{timetrap_timeout,TimetrapTimeout}} -> progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); + {died,Reason={auto_skip,_Why}} -> + %% died in init_per_testcase or in a hook in this context + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> %% died in init_per_testcase progress(skip, Num, Mod, Func, GrName, Loc, Reason, @@ -4798,6 +4860,14 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) -> {error,_Reason} = Error -> Error end; +collect_cases({repeat,{Module, Case}, Repeat}, St, Mode) -> + case catch collect_case([Case], St#cc{mod=Module}, [], Mode) of + {ok, [{Module,Case}], _} -> + {ok, [{repeat,{Module, Case}, Repeat}], St}; + Other -> + {error,Other} + end; + collect_cases({Module, Cases}, St, Mode) when is_list(Cases) -> case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of Result = {ok,_,_} -> @@ -5761,3 +5831,42 @@ encoding(File) -> E -> E end. + +check_repeat_testcase(Case,Result,Cases, + [{Ref,[{repeat,RepeatData0}],StartTime}|Mode0]) -> + case do_update_repeat_data(Result,RepeatData0) of + false -> + {Cases,Mode0}; + RepeatData -> + {[Case|Cases],[{Ref,[{repeat,RepeatData}],StartTime}|Mode0]} + end; +check_repeat_testcase(_,_,Cases,Mode) -> + {Cases,Mode}. + +do_update_repeat_data(_,{RT,N,N}) when is_integer(N) -> + report_repeat_testcase(N,N), + report_stop_repeat_testcase(done,{RT,N}), + false; +do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) -> + report_repeat_testcase(M,N), + report_stop_repeat_testcase(RT,{RT,N}), + false; +do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) -> + report_repeat_testcase(M,N), + report_stop_repeat_testcase(RT,{RT,N}), + false; +do_update_repeat_data(_,{RT,M,N}) when is_integer(M) -> + report_repeat_testcase(M,N), + {RT,M+1,N}; +do_update_repeat_data(_,{_,M,N}=RepeatData) -> + report_repeat_testcase(M,N), + RepeatData. + +report_stop_repeat_testcase(Reason,RepVal) -> + print(minor, "~n*** Stopping test case repeat operation: ~w", [Reason]), + print(1, "Stopping test case repeat operation: ~w", [RepVal]). + +report_repeat_testcase(M,forever) -> + print(minor, "~n=== Repeated test case: ~w of infinity", [M]); +report_repeat_testcase(M,N) -> + print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]). diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index ea7ad8538e..3ae4a047d8 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -656,9 +656,19 @@ find_release({unix,linux}, Rel) -> find_release(_, _) -> none. find_rel_linux(Rel) -> - case suse_release() of - none -> []; - SuseRel -> find_rel_suse(Rel, SuseRel) + try + case ubuntu_release() of + none -> none; + [UbuntuRel |_] -> throw(find_rel_ubuntu(Rel, UbuntuRel)) + end, + case suse_release() of + none -> none; + SuseRel -> throw(find_rel_suse(Rel, SuseRel)) + end, + [] + catch + throw:Result -> + Result end. find_rel_suse(Rel, SuseRel) -> @@ -735,6 +745,93 @@ suse_release(Fd) -> end end. +find_rel_ubuntu(_Rel, UbuntuRel) when is_integer(UbuntuRel), UbuntuRel < 16 -> + []; +find_rel_ubuntu(Rel, UbuntuRel) when is_integer(UbuntuRel) -> + Root = "/usr/local/otp/releases/ubuntu", + lists:foldl(fun (ChkUbuntuRel, Acc) -> + find_rel_ubuntu_aux1(Rel, Root++integer_to_list(ChkUbuntuRel)) + ++ Acc + end, + [], + lists:seq(16, UbuntuRel)). + +find_rel_ubuntu_aux1(Rel, RootWc) -> + case erlang:system_info(wordsize) of + 4 -> + find_rel_ubuntu_aux2(Rel, RootWc++"_32"); + 8 -> + find_rel_ubuntu_aux2(Rel, RootWc++"_64") ++ + find_rel_ubuntu_aux2(Rel, RootWc++"_32") + end. + +find_rel_ubuntu_aux2(Rel, RootWc) -> + RelDir = filename:dirname(RootWc), + Pat = filename:basename(RootWc ++ "_" ++ Rel) ++ ".*", + case file:list_dir(RelDir) of + {ok,Dirs} -> + case lists:filter(fun(Dir) -> + case re:run(Dir, Pat, [unicode]) of + nomatch -> false; + _ -> true + end + end, Dirs) of + [] -> + []; + [R|_] -> + [filename:join([RelDir,R,"bin","erl"])] + end; + _ -> + [] + end. + +ubuntu_release() -> + case file:open("/etc/lsb-release", [read]) of + {ok,Fd} -> + try + ubuntu_release(Fd, undefined, undefined) + after + file:close(Fd) + end; + {error,_} -> none + end. + +ubuntu_release(_Fd, DistrId, Rel) when DistrId /= undefined, + Rel /= undefined -> + Ubuntu = case DistrId of + "Ubuntu" -> true; + "ubuntu" -> true; + _ -> false + end, + case Ubuntu of + false -> none; + true -> Rel + end; +ubuntu_release(Fd, DistroId, Rel) -> + case io:get_line(Fd, '') of + eof -> + none; + Line when is_list(Line) -> + case re:run(Line, "^DISTRIB_ID=(\\w+)$", + [{capture,all_but_first,list}]) of + {match,[NewDistroId]} -> + ubuntu_release(Fd, NewDistroId, Rel); + nomatch -> + case re:run(Line, "^DISTRIB_RELEASE=(\\d+(?:\\.\\d+)*)$", + [{capture,all_but_first,list}]) of + {match,[RelList]} -> + NewRel = lists:map(fun (N) -> + list_to_integer(N) + end, + string:lexemes(RelList, ".")), + ubuntu_release(Fd, DistroId, NewRel); + nomatch -> + ubuntu_release(Fd, DistroId, Rel) + end + end + end. + + unpack(Bin) -> {One,Term} = split_binary(Bin, 1), case binary_to_list(One) of diff --git a/lib/common_test/src/test_server_sup.erl b/lib/common_test/src/test_server_sup.erl index 26e7534c6c..ab8066a88d 100644 --- a/lib/common_test/src/test_server_sup.erl +++ b/lib/common_test/src/test_server_sup.erl @@ -770,7 +770,7 @@ framework_call(Callback,Func,Args,DefaultReturn) -> end, case SetTcState of true -> - test_server:set_tc_state({framework,Mod,Func}); + test_server:set_tc_state({framework,{Mod,Func,Args}}); false -> ok end, diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index ecd1f727a2..e510b74d6a 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -74,7 +74,9 @@ MODULES= \ ct_SUITE \ ct_keep_logs_SUITE \ ct_unicode_SUITE \ - ct_auto_clean_SUITE + ct_auto_clean_SUITE \ + ct_util_SUITE \ + ct_tc_repeat_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_auto_compile_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE.erl index dface99b8f..f88f13c889 100644 --- a/lib/common_test/test/ct_auto_compile_SUITE.erl +++ b/lib/common_test/test/ct_auto_compile_SUITE.erl @@ -169,7 +169,7 @@ test_events(ac_flag) -> {?eh,start_info,{1,1,3}}, {?eh,tc_start,{ct_framework,error_in_suite}}, {?eh,tc_done,{ct_framework,error_in_suite, - {failed,{error,'bad_SUITE can not be compiled or loaded'}}}}, + {failed,{error,'bad_SUITE cannot be compiled or loaded'}}}}, {?eh,tc_start,{dummy_SUITE,init_per_suite}}, {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, {?eh,test_stats,{1,1,{1,0}}}, @@ -186,7 +186,7 @@ test_events(ac_spec) -> {?eh,start_info,{1,1,3}}, {?eh,tc_start,{ct_framework,error_in_suite}}, {?eh,tc_done,{ct_framework,error_in_suite, - {failed,{error,'bad_SUITE can not be compiled or loaded'}}}}, + {failed,{error,'bad_SUITE cannot be compiled or loaded'}}}}, {?eh,tc_start,{dummy_SUITE,init_per_suite}}, {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, {?eh,test_stats,{1,1,{1,0}}}, diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 7468ebe9d9..d31bd26273 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -648,33 +648,35 @@ test_events(cfg_error) -> {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, - {?eh,test_stats,{13,8,{0,19}}}, + {?eh,tc_done,{cfg_error_12_SUITE,tc1, + {auto_skipped, + {failed,{ct_framework,init_tc,{timetrap,500}}}}}}, + {?eh,test_stats,{13,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,8,{0,19}}}, + {?eh,test_stats,{14,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,8,{0,19}}}, + {?eh,test_stats,{15,7,{0,20}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,8,{0,19}}}, + {?eh,test_stats,{16,7,{0,20}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,8,{0,19}}}, + {?eh,test_stats,{17,7,{0,20}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,8,{0,19}}}, + {?eh,test_stats,{18,7,{0,20}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, @@ -728,25 +730,30 @@ test_events(lib_error) -> {lib_error_1_SUITE,no_lines_throw,{failed,{error,{thrown,catch_me_if_u_can}}}}}, {?eh,test_stats,{0,8,{0,0}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_error}}, - {?eh,tc_done,{ct_framework,init_tc, - {framework_error,{{badmatch,[1,2]},'_'}}}}, - {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_error, + {auto_skipped, + {failed, + {ct_framework,init_tc, + {{badmatch,[1,2]},'_'}}}}}}, + {?eh,test_stats,{0,8,{0,1}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_exit}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,byebye}}}, - {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_exit, + {auto_skipped,{failed,{ct_framework,init_tc,byebye}}}}}, + {?eh,test_stats,{0,8,{0,2}}}, {?eh,tc_start,{lib_error_1_SUITE,init_tc_throw}}, - {?eh,tc_done,{ct_framework,init_tc,{framework_error,catch_me_if_u_can}}}, - {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,init_tc_throw, + {auto_skipped,{failed,{ct_framework,init_tc, + catch_me_if_u_can}}}}}, + {?eh,test_stats,{0,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_error}}, - {?eh,tc_done,{ct_framework,end_tc, - {framework_error,{{badmatch,[1,2]},'_'}}}}, - {?eh,test_stats,{0,12,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_error,ok}}, % warning in comment + {?eh,test_stats,{1,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_exit}}, - {?eh,tc_done,{ct_framework,end_tc,{framework_error,byebye}}}, - {?eh,test_stats,{0,13,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_exit,ok}}, % warning in comment + {?eh,test_stats,{2,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_tc_throw}}, - {?eh,tc_done,{ct_framework,end_tc,{framework_error,catch_me_if_u_can}}}, - {?eh,test_stats,{0,14,{0,0}}}, + {?eh,tc_done,{lib_error_1_SUITE,end_tc_throw,ok}}, % warning in comment + {?eh,test_stats,{3,8,{0,3}}}, {?eh,tc_start,{lib_error_1_SUITE,end_per_suite}}, {?eh,tc_done,{lib_error_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 0f5636a789..340b8f3d52 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -73,18 +73,22 @@ all() -> all(suite) -> lists:reverse( [ + crash_groups, crash_all, bad_return_groups, bad_return_all, + illegal_values_groups, illegal_values_all, alter_groups, alter_all, + alter_all_to_skip, alter_all_from_skip, one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, faulty_cth_exit_in_init, faulty_cth_exit_in_id, faulty_cth_exit_in_init_scope_suite, minimal_cth, minimal_and_maximal_cth, faulty_cth_undef, scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, + scope_suite_group_only_cth, scope_per_suite_state_cth, scope_per_group_state_cth, scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, skip_pre_init_tc_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, - state_update_cth, options_cth, same_id_cth, + state_update_cth, update_result_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, no_init_suite_config, no_init_config, no_end_config, failed_sequence, repeat_force_stop, config_clash, @@ -152,6 +156,11 @@ scope_suite_cth(Config) when is_list(Config) -> do_test(scope_suite_cth, "ct_scope_suite_cth_SUITE.erl", [],Config). +scope_suite_group_only_cth(Config) when is_list(Config) -> + do_test(scope_suite_group_only_cth, + "ct_scope_suite_group_only_cth_SUITE.erl", + [],Config,ok,2,[{group,g1}]). + scope_per_group_cth(Config) when is_list(Config) -> do_test(scope_per_group_cth, "ct_scope_per_group_cth_SUITE.erl", [],Config). @@ -209,6 +218,10 @@ state_update_cth(Config) when is_list(Config) -> do_test(state_update_cth, "ct_cth_fail_one_skip_one_SUITE.erl", [state_update_cth,state_update_cth],Config). +update_result_cth(Config) -> + do_test(update_result_cth, "ct_cth_update_result_post_end_tc_SUITE.erl", + [update_result_post_end_tc_cth],Config). + options_cth(Config) when is_list(Config) -> do_test(options_cth, "ct_cth_empty_SUITE.erl", [{empty_cth,[test]}],Config). @@ -300,10 +313,74 @@ repeat_force_stop(Config) -> [{force_stop,skip_rest},{duration,"000009"}]). %% Test that expected callbacks, and only those, are called when a test -%% are fails due to clash in config alias names +%% fails due to clash in config alias names config_clash(Config) -> do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config). +%% Test post_groups and post_all hook callbacks, introduced by OTP-14746 +alter_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_groups_return,[{new_group,[tc1,tc2]}]}, + {post_all_return,[{group,new_group}]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,[tc2]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all_from_skip(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{all_return,{skip,"skipped by all/0"}}, + {post_all_return,[tc2]}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +alter_all_to_skip(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_all_return,{skip,"skipped by post_all/3"}}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +bad_return_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,not_a_list}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +bad_return_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,not_a_list}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +illegal_values_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_groups_return,[{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +illegal_values_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_all_return,[{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}]}], + Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +crash_groups(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,crash}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + +crash_all(Config) -> + CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,crash}],Config), + do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth], + Config, ok, 2, [{config,CfgFile}]). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -323,6 +400,7 @@ do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) -> filename:join([DataDir,"cth/tests",Wildcard])), {Opts,ERPid} = setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config), + Res = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), @@ -348,6 +426,13 @@ reformat(Events, EH) -> %reformat(Events, _EH) -> % Events. +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- @@ -366,13 +451,16 @@ test_events(one_empty_cth) -> {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{empty_cth,id,[[]]}}, {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + %% check that post_groups and post_all comes after init when hook + %% is installed with start flag/option. + {?eh,cth,{empty_cth,post_groups,[ct_cth_empty_SUITE,[]]}}, + {?eh,cth,{empty_cth,post_all,[ct_cth_empty_SUITE,[test_case],[]]}}, {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{empty_cth,pre_init_per_suite, [ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_suite, [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, - {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, @@ -581,6 +669,10 @@ test_events(scope_suite_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + %% check that post_groups and post_all comes before init when hook + %% is installed in suite/0 + {?eh,cth,{'_',post_groups,['_',[]]}}, + {?eh,cth,{'_',post_all,['_','_',[]]}}, {?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, @@ -602,6 +694,34 @@ test_events(scope_suite_cth) -> {?eh,stop_logging,[]} ]; +test_events(scope_suite_group_only_cth) -> + Suite = ct_scope_suite_group_only_cth_SUITE, + CTH = empty_cth, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,1}}, + %% check that post_groups and post_all comes before init when hook + %% is installed in suite/0 + {?eh,cth,{CTH,post_groups,['_',['_']]}}, + {negative, + {?eh,cth,{CTH,post_all,['_','_','_']}}, + {?eh,tc_start,{Suite,init_per_suite}}}, + {?eh,cth,{CTH,id,[[]]}}, + {?eh,cth,{CTH,init,['_',[]]}}, + {?eh,cth,{CTH,pre_init_per_suite,[Suite,'$proplist',mystate]}}, + {?eh,cth,{CTH,post_init_per_suite,[Suite,'$proplist','$proplist',mystate]}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,cth,{CTH,pre_end_per_suite,[Suite,'$proplist',mystate]}}, + {?eh,cth,{CTH,post_end_per_suite,[Suite,'$proplist','_',mystate]}}, + {?eh,cth,{CTH,terminate,[mystate]}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(scope_per_group_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -661,6 +781,8 @@ test_events(scope_suite_state_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{'_',post_groups,['_',[]]}}, + {?eh,cth,{'_',post_all,['_','_',[]]}}, {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,init_per_suite}}, {?eh,cth,{'_',id,[[test]]}}, {?eh,cth,{'_',init,['_',[test]]}}, @@ -1099,6 +1221,106 @@ test_events(state_update_cth) -> {?eh,stop_logging,[]} ]; +test_events(update_result_cth) -> + Suite = ct_cth_update_result_post_end_tc_SUITE, + [ + {?eh,start_logging,'_'}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,tc_start,{Suite,init_per_suite}}, + {?eh,tc_done,{Suite,init_per_suite,ok}}, + + {?eh,tc_start,{Suite,tc_ok_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_fail,'_',ok,[]]}}, + {?eh,tc_done,{Suite,tc_ok_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{0,1,{0,0}}}, + + {?eh,tc_start,{Suite,tc_ok_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase,[Suite,tc_ok_to_skip,'_',ok,[]]}}, + {?eh,tc_done,{Suite,tc_ok_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{0,1,{1,0}}}, + + {?eh,tc_start,{Suite,tc_fail_to_ok}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_fail_to_ok,'_', + {error,{test_case_failed,"should be changed to ok"}},[]]}}, + {?eh,tc_done,{Suite,tc_fail_to_ok,ok}}, + {?eh,test_stats,{1,1,{1,0}}}, + + {?eh,tc_start,{Suite,tc_fail_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_fail_to_skip,'_', + {error,{test_case_failed,"should be changed to skip"}},[]]}}, + {?eh,tc_done,{Suite,tc_fail_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{1,1,{2,0}}}, + + {?eh,tc_start,{Suite,tc_timetrap_to_ok}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_timetrap_to_ok,'_',{timetrap_timeout,3000},[]]}}, + {?eh,tc_done,{Suite,tc_timetrap_to_ok,ok}}, + {?eh,test_stats,{2,1,{2,0}}}, + + {?eh,tc_start,{Suite,tc_timetrap_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_timetrap_to_skip,'_',{timetrap_timeout,3000},[]]}}, + {?eh,tc_done,{Suite,tc_timetrap_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{2,1,{3,0}}}, + + {?eh,tc_start,{Suite,tc_skip_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,tc_skip_to_fail,'_', + {skip,"should be changed to fail"},[]]}}, + {?eh,tc_done,{Suite,tc_skip_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{2,2,{3,0}}}, + + {?eh,tc_start,{Suite,end_fail_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_fail_to_fail,'_', + {failed, + {Suite,end_per_testcase, + {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}}, + {?eh,tc_done,{Suite,end_fail_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{2,3,{3,0}}}, + + {?eh,tc_start,{Suite,end_fail_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_fail_to_skip,'_', + {failed, + {Suite,end_per_testcase, + {'EXIT',{test_case_failed,"change result when end fails"}}}},[]]}}, + {?eh,tc_done,{Suite,end_fail_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{2,3,{4,0}}}, + + {?eh,tc_start,{Suite,end_timetrap_to_fail}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_timetrap_to_fail,'_', + {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}}, + {?eh,tc_done,{Suite,end_timetrap_to_fail,{failed,{error,"Test failure"}}}}, + {?eh,cth,{'_',on_tc_fail,'_'}}, + {?eh,test_stats,{2,4,{4,0}}}, + + {?eh,tc_start,{Suite,end_timetrap_to_skip}}, + {?eh,cth,{'_',post_end_per_testcase, + [Suite,end_timetrap_to_skip,'_', + {failed,{Suite,end_per_testcase,{timetrap_timeout,3000}}},[]]}}, + {?eh,tc_done,{Suite,end_timetrap_to_skip,{skipped,"Test skipped"}}}, + {?eh,cth,{'_',on_tc_skip,'_'}}, + {?eh,test_stats,{2,4,{5,0}}}, + + {?eh,tc_start,{Suite,end_per_suite}}, + {?eh,tc_done,{Suite,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(options_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -2209,6 +2431,229 @@ test_events(config_clash) -> %% Make sure no 'cth_error' events are received! [{negative,{?eh,cth_error,'_'},E} || E <- Events]; +test_events(alter_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{new_group,[tc1,tc2]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}], + [{new_group,[tc1,tc2]}]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{new_group,[tc1,tc2]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}], + [{new_group,[tc1,tc2]}]]}}, + {?eh,tc_start,{all_and_groups_SUITE,{init_per_group,new_group,[]}}}, + {?eh,tc_done,{all_and_groups_SUITE, + {init_per_group,new_group,'$proplist'},ok}}, + {?eh,tc_start,{all_and_groups_SUITE,tc1}}, + {?eh,tc_done,{all_and_groups_SUITE,tc1,ok}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,tc_start,{all_and_groups_SUITE,{end_per_group,new_group,[]}}}, + {?eh,tc_done,{all_and_groups_SUITE, + {end_per_group,new_group,'$proplist'},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2], + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all_from_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2], + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}}, + {?eh,tc_start,{all_and_groups_SUITE,tc2}}, + {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(alter_all_to_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE, + [{test_group,[tc1]}]]}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE, + {skip,"skipped by post_all/3"}, + [{test_group,[tc1]}]]}}, + {?eh,start_info,{1,1,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE, + {skip,"skipped by post_all/3"}, + '_']}}, + {?eh,tc_user_skip,{all_and_groups_SUITE,all,"skipped by post_all/3"}}, + {?eh,cth,{'_',on_tc_skip,[all_and_groups_SUITE,all, + {tc_user_skip,"skipped by post_all/3"}, + []]}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(illegal_values_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups, + [all_and_groups_SUITE, + [{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups, + [all_and_groups_SUITE, + [{new_group,[this_test_does_not_exist]}, + this_is_not_a_group_def]]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite,{failed,{error,'_'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(illegal_values_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all, + [all_and_groups_SUITE, + [{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}],'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all, + [all_and_groups_SUITE, + [{group,this_group_does_not_exist}, + {this_is_not_a_valid_term}],'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error,'Invalid reference to group this_group_does_not_exist in all_and_groups_SUITE:all/0'}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(bad_return_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error, + {'Bad return value from post_groups/2 hook function',not_a_list}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(bad_return_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done, + {ct_framework,error_in_suite, + {failed, + {error,{'Bad return value from post_all/3 hook function',not_a_list}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(crash_groups) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed, + {error,"all_and_groups_cth:post_groups/2 CTH call failed"}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(crash_all) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}}, + {?eh,start_info,{1,0,0}}, + {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}}, + {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed, + {error,"all_and_groups_cth:post_all/3 CTH call failed"}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(ok) -> ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl new file mode 100644 index 0000000000..adc86005f9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl @@ -0,0 +1,47 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(all_and_groups_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_group(_Group,Config) -> + Config. + +end_per_group(_Group,Config) -> + ok. + +all() -> + ct:get_config(all_return,[{group,test_group}]). + +groups() -> + [{test_group,[tc1]}]. + +%% Test cases starts here. +tc1(Config) -> + ok. + +tc2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl new file mode 100644 index 0000000000..9ebc00e9de --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl @@ -0,0 +1,100 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(all_and_groups_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments +-define(fail(Info), + gen_event:notify( + ?CT_EVMGR_REF, + #event{ name = cth_error, + node = node(), + data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})). + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +post_groups(Suite,Groups) -> + case empty_cth:post_groups(Suite,ct:get_config(post_groups_return,Groups)) of + crash -> error(crash_in_post_groups); + R -> R + end. + +post_all(Suite,Tests,Groups) -> + case empty_cth:post_all(Suite,ct:get_config(post_all_return,Tests),Groups) of + crash -> error(crash_in_post_all); + R -> R + end. + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC,Reason,State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC,Reason,State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl new file mode 100644 index 0000000000..a16138ce6f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_cth_update_result_post_end_tc_SUITE.erl @@ -0,0 +1,101 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_cth_update_result_post_end_tc_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + [{timetrap,{seconds,3}}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(EndTimetrap,_) when EndTimetrap==end_timetrap_to_fail; + EndTimetrap==end_timetrap_to_skip-> + timer:sleep(10000); +end_per_testcase(EndFail,_) when EndFail==end_fail_to_fail; + EndFail==end_fail_to_skip-> + ct:fail("change result when end fails"); +end_per_testcase(_,_) -> + ok. + +all() -> + [tc_ok_to_fail, + tc_ok_to_skip, + tc_fail_to_ok, + tc_fail_to_skip, + tc_timetrap_to_ok, + tc_timetrap_to_skip, + tc_skip_to_fail, + end_fail_to_fail, + end_fail_to_skip, + end_timetrap_to_fail, + end_timetrap_to_skip]. + +%% Test cases starts here. +tc_ok_to_fail(_Config) -> + ok. + +tc_ok_to_skip(_Config) -> + ok. + +tc_fail_to_ok(_Config) -> + ct:fail("should be changed to ok"). + +tc_fail_to_skip(_Config) -> + ct:fail("should be changed to skip"). + +tc_timetrap_to_ok(_Config) -> + timer:sleep(10000), % will time out after 3 sek + ok. + +tc_timetrap_to_skip(_Config) -> + timer:sleep(10000), % will time out after 3 sek + ok. + +tc_skip_to_fail(_Config) -> + {skip,"should be changed to fail"}. + +end_fail_to_fail(_Config) -> + ok. + +end_fail_to_skip(_Config) -> + ok. + +end_timetrap_to_fail(_Config) -> + ok. + +end_timetrap_to_skip(_Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl new file mode 100644 index 0000000000..38c9da903d --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(ct_match_state_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +post_groups(Suite, Groups) -> + empty_cth:post_groups(Suite, Groups). + +post_all(Suite, Tests, Groups) -> + empty_cth:post_all(Suite, Tests, Groups). + +init(Id, Opts) -> + empty_cth:init(Id, Opts), + {ok,mystate}. + +%% In the following, always match against the state value, to ensure +%% that init has indeed been called before the rest of the hooks. +pre_init_per_suite(Suite,Config,mystate) -> + empty_cth:pre_init_per_suite(Suite,Config,mystate). + +post_init_per_suite(Suite,Config,Return,mystate) -> + empty_cth:post_init_per_suite(Suite,Config,Return,mystate). + +pre_end_per_suite(Suite,Config,mystate) -> + empty_cth:pre_end_per_suite(Suite,Config,mystate). + +post_end_per_suite(Suite,Config,Return,mystate) -> + empty_cth:post_end_per_suite(Suite,Config,Return,mystate). + +terminate(mystate) -> + empty_cth:terminate(mystate). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl new file mode 100644 index 0000000000..537c97d3f0 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl @@ -0,0 +1,54 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_scope_suite_group_only_cth_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +suite() -> + [{ct_hooks,[ct_match_state_cth]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [test_case]. + +groups() -> + [{g1,[test_case]}]. + +%% Test cases starts here. +test_case(Config) when is_list(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl index c648367838..60488e84c6 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -39,6 +39,9 @@ -export([id/1]). -export([init/2]). +-export([post_all/3]). +-export([post_groups/2]). + -export([pre_init_per_suite/3]). -export([post_init_per_suite/4]). -export([pre_end_per_suite/3]). @@ -71,6 +74,31 @@ -record(state, { id = ?MODULE :: term()}). +%% Called after groups/0. +%% You can change the return value in this function. +-spec post_groups(Suite :: atom(), Groups :: list()) -> list(). +post_groups(Suite,Groups) -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_groups, + [Suite,Groups]}}), + ct:log("~w:post_groups(~w) called", [?MODULE,Suite]), + Groups. + +%% Called after all/0. +%% You can change the return value in this function. +-spec post_all(Suite :: atom(), + Tests :: list(), + Groups :: term()) -> + list(). +post_all(Suite,Tests,Groups) -> + gen_event:notify( + ?CT_EVMGR_REF, #event{ name = cth, node = node(), + data = {?MODULE, post_all, + [Suite,Tests,Groups]}}), + ct:log("~w:post_all(~w) called", [?MODULE,Suite]), + Tests. + %% Always called before any other callback function. Use this to initiate %% any common state. It should return an state for this CTH. -spec init(Id :: term(), Opts :: proplists:proplist()) -> diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl new file mode 100644 index 0000000000..7afb3d8781 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_result_post_end_tc_cth.erl @@ -0,0 +1,98 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(update_result_post_end_tc_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State), + change_result(TC,Config,State). + +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). + +%%%----------------------------------------------------------------- +%%% +change_result(tc_ok_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(tc_ok_to_skip,_Config,State) -> + {{skip, "Test skipped"}, State}; +change_result(tc_fail_to_ok,Config,State) -> + {lists:keydelete(tc_status,1,Config),State}; +change_result(tc_fail_to_skip,Config,State) -> + {{skip,"Test skipped"},State}; +change_result(tc_timetrap_to_ok,Config,State) -> + {lists:keydelete(tc_status,1,Config),State}; +change_result(tc_timetrap_to_skip,Config,State) -> + {{skip,"Test skipped"},State}; +change_result(tc_skip_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(end_fail_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(end_fail_to_skip,_Config,State) -> + {{skip, "Test skipped"}, State}; +change_result(end_timetrap_to_fail,_Config,State) -> + {{fail, "Test failure"}, State}; +change_result(end_timetrap_to_skip,_Config,State) -> + {{skip, "Test skipped"}, State}. diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index a2fa099a8c..0d17481e95 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -440,6 +440,12 @@ edit_config(Config) -> ?ok = ct_netconfc:edit_config(Client,running, {server,[{xmlns,"myns"}], [{name,["myserver"]}]}), + ?NS:expect_reply('edit-config',ok), + ?ok = ct_netconfc:edit_config(Client,running, + [{server,[{xmlns,"myns"}], + [{name,["server1"]}]}, + {server,[{xmlns,"myns"}], + [{name,["server2"]}]}]), ?NS:expect_do_reply('close-session',close,ok), ?ok = ct_netconfc:close_session(Client), ok. diff --git a/lib/common_test/test/ct_tc_repeat_SUITE.erl b/lib/common_test/test/ct_tc_repeat_SUITE.erl new file mode 100644 index 0000000000..433b5456fe --- /dev/null +++ b/lib/common_test/test/ct_tc_repeat_SUITE.erl @@ -0,0 +1,438 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_tc_repeat_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{minutes,1}}]. + +all() -> + all(suite). + +all(suite) -> + [ + repeat, + repeat_parallel_until_ok, + repeat_parallel_until_fail, + repeat_sequence_until_ok, + repeat_sequence_until_fail, + pick_one_test_from_group, + pick_one_test_from_subgroup + ]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +%% Test post_groups and post_all hook callbacks, introduced by OTP-14746 +repeat(Config) -> + ok = do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [], [], Config). + +repeat_parallel_until_ok(Config) -> + {error,{{illegal_combination,{parallel,repeat_until_ok}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_ok}], + [], Config, 1, []). + +repeat_parallel_until_fail(Config) -> + {error,{{illegal_combination,{parallel,repeat_until_fail}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_fail}], + [], Config, 1, []). + +repeat_sequence_until_ok(Config) -> + {error,{{illegal_combination,{sequence,repeat_until_ok}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_ok}], + [], Config, 1, []). + +repeat_sequence_until_fail(Config) -> + {error,{{illegal_combination,{sequence,repeat_until_fail}},_}} = + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_fail}], + [], Config, 1, []). + +pick_one_test_from_group(Config) -> + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_mixed},{testcase,tc2}], + [], Config, 1, []). + +pick_one_test_from_subgroup(Config) -> + do_test(?FUNCTION_NAME, "tc_repeat_SUITE", + [{group,[[g_mixed,subgroup]]},{testcase,tc2}], + [], Config, 1, []). + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, Suite, WTT, CTHs, Config) -> + do_test(Tag, Suite, WTT, CTHs, Config, 2, []). + +do_test(Tag, Suite0, WTT, CTHs, Config, EC, ExtraOpts) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join([DataDir,Suite0]), + {Opts,ERPid} = + setup([{suite,Suite}|WTT]++[{ct_hooks,CTHs},{label,Tag}|ExtraOpts], + Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + %% io:format("~p~n",[Events]), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + Res. + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(repeat) -> + S = tc_repeat_SUITE, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,unknown}}, + + %% tc1, {repeat,2} + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + %% tc2, {repeat_until_ok,3} + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + %% tc3, {repeat_until_ok,3} + {?eh,tc_start,{S,tc3}}, + {?eh,tc_done,{tc_repeat_SUITE,tc3, + {failed,{error,{test_case_failed,always_fail}}}}}, + {?eh,test_stats,{3,1,{0,0}}}, + {?eh,tc_start,{S,tc3}}, + {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}}, + {?eh,test_stats,{3,2,{0,0}}}, + {?eh,tc_start,{S,tc3}}, + {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}}, + {?eh,test_stats,{3,3,{0,0}}}, + %% tc4, {repeat_until_fail,3} + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,ok}}, + {?eh,test_stats,{4,3,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, + {?eh,test_stats,{4,4,{0,0}}}, + %% g, tc1, {repeat,2} + {?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{5,4,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{6,4,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}, + %% g_until_ok, tc2, {repeat_until_ok,3} + {?eh,tc_start,{S,{init_per_group,g_until_ok,[]}}}, + {?eh,tc_done,{S,{init_per_group,g_until_ok,[]},ok}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{7,4,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g_until_ok,[]}}}, + {?eh,tc_done,{S,{end_per_group,g_until_ok,[]},ok}}, + %% g_until_fail, tc4, {repeat_until_fail,3} + {?eh,tc_start,{S,{init_per_group,g_until_fail,[]}}}, + {?eh,tc_done,{S,{init_per_group,g_until_fail,[]},ok}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,ok}}, + {?eh,test_stats,{8,4,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, + {?eh,test_stats,{8,5,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g_until_fail,[]}}}, + {?eh,tc_done,{S,{end_per_group,g_until_fail,[]},ok}}, + %% g, parallel, tc1, {repeat,2} + {parallel, + [{?eh,tc_start,{S,{init_per_group,g,[parallel]}}}, + {?eh,tc_done,{S,{init_per_group,g,[parallel]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{9,5,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{10,5,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[parallel]}}}, + {?eh,tc_done,{S,{end_per_group,g,[parallel]},ok}}]}, + %% g, sequence, tc1, {repeat,2} + {?eh,tc_start,{S,{init_per_group,g,[sequence]}}}, + {?eh,tc_done,{S,{init_per_group,g,[sequence]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{11,5,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{12,5,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[sequence]}}}, + {?eh,tc_done,{S,{end_per_group,g,[sequence]},ok}}, + %% g_sequence_skip_rest, + {?eh,tc_start,{S,{init_per_group,g_mixed,[sequence]}}}, + {?eh,tc_done,{S,{init_per_group,g_mixed,[sequence]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{13,5,{0,0}}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{14,5,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,ok}}, + {?eh,test_stats,{15,5,{0,0}}}, + {?eh,tc_start,{S,tc4}}, + {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}}, + {?eh,test_stats,{15,6,{0,0}}}, + %% ----> fail in sequence, so skip rest + {?eh,tc_auto_skip,{S,{tc4,g_mixed}, % last of current repeat tc4 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,1}}}, + {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,2}}}, + {?eh,tc_auto_skip,{S,{tc1,g}, % group g, tc1, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,3}}}, + {?eh,tc_auto_skip,{S,{tc1,subgroup}, % subgroup, single tc1 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,4}}}, + {?eh,tc_auto_skip,{S,{tc2,subgroup}, % subgroup, tc2, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,5}}}, + {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % tc2, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,6}}}, + {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % single tc2 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,7}}}, + {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % tc1, {repeat,2} + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,8}}}, + {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1 + {failed,{tc_repeat_SUITE,tc4}}}}, + {?eh,test_stats,{15,6,{0,9}}}, + {?eh,tc_start,{S,{end_per_group,g_mixed,'_'}}}, + {?eh,tc_done,{S,{end_per_group,g_mixed,'_'},ok}}, + %% done + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_parallel_until_ok) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_parallel_until_ok,[parallel]}}}, + {?eh,tc_done,{'_',{init_per_group,g_parallel_until_ok,[parallel]},ok}}, + {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_ok}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_parallel_until_fail) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_parallel_until_fail,[parallel]}}}, + {?eh,tc_done,{'_',{init_per_group,g_parallel_until_fail,[parallel]},ok}}, + {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_fail}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_sequence_until_ok) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_sequence_until_ok,[sequence]}}}, + {?eh,tc_done,{'_',{init_per_group,g_sequence_until_ok,[sequence]},ok}}, + {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_ok}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(repeat_sequence_until_fail) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_sequence_until_fail,[sequence]}}}, + {?eh,tc_done,{'_',{init_per_group,g_sequence_until_fail,[sequence]},ok}}, + {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_fail}},'_'}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(pick_one_test_from_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}}, + {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',tc2}}}, % single tc2 + {?eh,tc_done,{'_',tc2,ok}}, + {?eh,tc_start,{'_',tc2}}, % tc2, {repeat,2} + {?eh,tc_done,{'_',tc2,ok}}, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_done,{'_',tc2,ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}}, + {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(pick_one_test_from_subgroup) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}}, + {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_start,{'_',{init_per_group,subgroup,[]}}}}, + {?eh,tc_done,{'_',{init_per_group,subgroup,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',tc2}}}, % tc2, {repeat,2} + {?eh,tc_done,{'_',tc2,ok}}, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_done,{'_',tc2,ok}}, + {negative, + {?eh,tc_start,{'_',tc1}}, + {?eh,tc_start,{'_',{end_per_group,subgroup,[]}}}}, + {?eh,tc_done,{'_',{end_per_group,subgroup,[]},ok}}, + {negative, + {?eh,tc_start,{'_',tc2}}, + {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}}, + {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(ok) -> + ok. + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl b/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl new file mode 100644 index 0000000000..f5d960d12f --- /dev/null +++ b/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl @@ -0,0 +1,85 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(tc_repeat_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_group(_Group,Config) -> + Config. + +end_per_group(_Group,Config) -> + ok. + +all() -> + [{testcase,tc1,[{repeat,2}]}, + {testcase,tc2,[{repeat_until_ok,3}]}, + {testcase,tc3,[{repeat_until_ok,3}]}, + {testcase,tc4,[{repeat_until_fail,3}]}, + {group,g}, + {group,g_until_ok}, + {group,g_until_fail}, + {group,g,[parallel]}, + {group,g,[sequence]}, + {group,g_mixed,[sequence]}]. + +groups() -> + [{g,[{testcase,tc1,[{repeat,2}]}]}, + {g_until_ok,[{testcase,tc2,[{repeat_until_ok,3}]}]}, + {g_until_fail,[{testcase,tc4,[{repeat_until_fail,3}]}]}, + {g_parallel_until_ok,[parallel],[{testcase,tc2,[{repeat_until_ok,3}]}]}, + {g_parallel_until_fail,[parallel],[{testcase,tc1,[{repeat_until_fail,2}]}]}, + {g_sequence_until_ok,[sequence],[{testcase,tc2,[{repeat_until_ok,3}]}]}, + {g_sequence_until_fail,[sequence],[{testcase,tc1,[{repeat_until_fail,2}]}]}, + {g_mixed,[{testcase,tc1,[{repeat,2}]}, + {testcase,tc4,[{repeat,3}]}, + tc1, + {group,g}, + {subgroup,[tc1,{testcase,tc2,[{repeat,2}]}]}, + {testcase,tc2,[{repeat,2}]}, + tc2, + {testcase,tc1,[{repeat,2}]}, + tc1]}]. + +%% Test cases starts here. +tc1(_Config) -> + ok. + +tc2(_Config) -> + ok. + +tc3(_Config) -> + ct:fail(always_fail). + +tc4(Config) -> + case ?config(saved_config,Config) of + {tc4,_} -> + ct:fail(second_time_fail); + undefined -> + {save_config,Config} + end. + +tc5(_Config) -> + {skip,"just skip this"}. diff --git a/lib/common_test/test/ct_telnet_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE.erl index a0089c9bc9..f71b7c370f 100644 --- a/lib/common_test/test/ct_telnet_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE.erl @@ -50,10 +50,10 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. groups() -> - [{legacy, [], [unix_telnet,own_server,timetrap]}, - {raw, [], [unix_telnet,own_server,timetrap]}, - {html, [], [unix_telnet,own_server]}, - {silent, [], [unix_telnet,own_server]}]. + [{legacy, [], [unix_telnet,own_server,faulty_regexp,timetrap]}, + {raw, [], [unix_telnet,own_server,faulty_regexp,timetrap]}, + {html, [], [unix_telnet,own_server,faulty_regexp]}, + {silent, [], [unix_telnet,own_server,faulty_regexp]}]. all() -> [ @@ -119,6 +119,12 @@ own_server(Config) -> all_tests_in_suite(own_server,"ct_telnet_own_server_SUITE", CfgFile,Config). +faulty_regexp(Config) -> + CfgFile = "telnet.faulty_regexp." ++ + atom_to_list(groupname(Config)) ++ ".cfg", + all_tests_in_suite(faulty_regexp,"ct_telnet_faulty_regexp_SUITE", + CfgFile,Config). + timetrap(Config) -> CfgFile = "telnet.timetrap." ++ atom_to_list(groupname(Config)) ++ ".cfg", @@ -225,6 +231,31 @@ events_to_check(unix_telnet,Config) -> all_cases(ct_telnet_basic_SUITE,Config); events_to_check(own_server,Config) -> all_cases(ct_telnet_own_server_SUITE,Config); +events_to_check(faulty_regexp,_Config) -> + [{?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_pattern, + {failed, + {error,{{bad_pattern,"invalid(pattern",{"missing )",15}}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_pattern_no_string, + {failed, + {error,{{bad_pattern,invalid_pattern}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_tag_pattern, + {failed, + {error,{{bad_pattern,{tag,"invalid(pattern"},{"missing )",15}}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done, + {ct_telnet_faulty_regexp_SUITE,expect_tag_pattern_no_string, + {failed, + {error,{{bad_pattern,{tag,invalid_pattern}}, + {ct_telnet,expect,3}}}}}}, + {?eh,tc_done,{ct_telnet_faulty_regexp_SUITE,expect_pattern_unicode,ok}}, + {?eh,tc_done,{ct_telnet_faulty_regexp_SUITE,expect_tag_pattern_unicode,ok}}, + {?eh,stop_logging,[]}]; events_to_check(timetrap,_Config) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,tc_done,{ct_telnet_timetrap_SUITE,expect_timetrap, diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_faulty_regexp_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_faulty_regexp_SUITE.erl new file mode 100644 index 0000000000..a5c9451a9c --- /dev/null +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_faulty_regexp_SUITE.erl @@ -0,0 +1,79 @@ +-module(ct_telnet_faulty_regexp_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(name, telnet_server_conn1). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +suite() -> [{require,?name,{unix,[telnet]}}, + {require,ct_conn_log}, + {ct_hooks, [{cth_conn_log,[]}]}]. + +all() -> + [expect_pattern, + expect_pattern_no_string, + expect_tag_pattern, + expect_tag_pattern_no_string, + expect_pattern_unicode, + expect_tag_pattern_unicode]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +init_per_testcase(_,Config) -> + ct:log("init_per_testcase: opening telnet connection...",[]), + {ok,_} = ct_telnet:open(?name), + ct:log("...done",[]), + Config. + +end_per_testcase(_,_Config) -> + ct:log("end_per_testcase: closing telnet connection...",[]), + _ = ct_telnet:close(?name), + ct:log("...done",[]), + ok. + +expect_pattern(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, "invalid(pattern"). + +expect_pattern_no_string(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, invalid_pattern). + +expect_tag_pattern(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, {tag,"invalid(pattern"}). + +expect_tag_pattern_no_string(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + ok = ct_telnet:expect(?name, {tag,invalid_pattern}). + +%% Test that a unicode pattern can be given without the testcase +%% failing. Do however notice that there is no real unicode support +%% in ct_telnet yet, that is, the telnet binary mode is not supported. +expect_pattern_unicode(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + {error,{prompt,_}} = ct_telnet:expect(?name, "pattern_with_unicode_αβ"), + ok. + +expect_tag_pattern_unicode(_) -> + ok = ct_telnet:send(?name, "echo ayt"), + {error,{prompt,_}} = ct_telnet:expect(?name, "pattern_with_unicode_αβ"), + ok. diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl index 985fa40ad2..34df57027e 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl @@ -58,7 +58,8 @@ all() -> server_speaks, server_disconnects, newline_ayt, - newline_break + newline_break, + newline_string ]. groups() -> @@ -393,3 +394,11 @@ newline_break(_) -> "> " = lists:flatten(R), ok = ct_telnet:close(Handle), ok. + +%% Test option {newline,String} to specify an own newline, e.g. "\r\n" +newline_string(_) -> + {ok, Handle} = ct_telnet:open(telnet_server_conn1), + ok = ct_telnet:send(Handle, "echo hello-", [{newline,"own_nl\n"}]), + {ok,["hello-own_nl"]} = ct_telnet:expect(Handle, ["hello-own_nl"]), + ok = ct_telnet:close(Handle), + ok. diff --git a/lib/common_test/test/ct_util_SUITE.erl b/lib/common_test/test/ct_util_SUITE.erl new file mode 100644 index 0000000000..1d773855da --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE.erl @@ -0,0 +1,490 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_util_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + DataDir = ?config(data_dir, Config), + CTHs = filelib:wildcard(filename:join(DataDir,"*_cth.erl")), + io:format("CTHs: ~p",[CTHs]), + [io:format("Compiling ~p: ~p", + [FileName,compile:file(FileName,[{outdir,DataDir},debug_info])]) || + FileName <- CTHs], + ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + + +suite() -> + [{timetrap,{minutes,1}}]. + +all() -> + all(suite). + +all(suite) -> + [ + pre_init_per_suite, + post_init_per_suite, + pre_end_per_suite, + post_end_per_suite, + pre_init_per_group, + post_init_per_group, + pre_end_per_group, + post_end_per_group, + pre_init_per_testcase, + post_init_per_testcase, + pre_end_per_testcase, + post_end_per_testcase + ]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% +pre_init_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_suite(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_suite, + {curr_tc_SUITE,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + + +pre_init_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_group(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_group, + {curr_tc_SUITE,g,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_init_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_init_per_testcase, + {curr_tc_SUITE,tc1,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_init_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_init_per_testcase, + {curr_tc_SUITE,tc1,{timeout,5000}}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +pre_end_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{pre_end_per_testcase, + {curr_tc_SUITE,tc1,{timeout,5000}}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +post_end_per_testcase(Config) -> + CfgFile = gen_config(?FUNCTION_NAME, + [{post_end_per_testcase, + {curr_tc_SUITE,tc1,kill}}], + Config), + ok = do_test(?FUNCTION_NAME, + "curr_tc_SUITE.erl", + [{ct_hooks,[ct_util_cth]},{config,CfgFile}], + Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +do_test(Tag, Suite, RunTestArgs, Config) -> + do_test(Tag, Suite, RunTestArgs, Config, 2). + +do_test(Tag, Suite0, RunTestArgs, Config, EC) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join([DataDir,Suite0]), + {Opts,ERPid} = setup([{suite,Suite}]++[{label,Tag}|RunTestArgs],Config), + Res = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + %% io:format("~p~n",[Events]), + + ct_test_support:log_events(Tag, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(Tag, EC), + ok = ct_test_support:verify_events(TestEvents, Events, Config), + Res. + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +gen_config(Name,KeyVals,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,atom_to_list(Name)++".cfg"), + ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}]) + || {Key,Value} <- KeyVals]), + File. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(IPS) when IPS=:=pre_init_per_suite; IPS=:=post_init_per_suite -> + S = curr_tc_SUITE, + FwFunc = + case IPS of + pre_init_per_suite -> init_tc; + post_init_per_suite -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,E}}, + {?eh,tc_auto_skip,{S,tc1,{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,tc2,{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_suite,E}}}}, + {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_suite,E}}}}, + {?eh,test_stats,{0,0,{0,4}}}, + {?eh,tc_auto_skip,{S,end_per_suite,{failed,{S,init_per_suite,E}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPS) when EPS=:=pre_end_per_suite; EPS=:=post_end_per_suite -> + S = curr_tc_SUITE, + FwFunc = + case EPS of + pre_end_per_suite -> init_tc; + post_end_per_suite -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,E}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(IPG) when IPG=:=pre_init_per_group; IPG=:=post_init_per_group -> + S = curr_tc_SUITE, + FwFunc = + case IPG of + pre_init_per_group -> init_tc; + post_init_per_group -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},E}}, + {?eh,tc_auto_skip,{S,{tc1,g},{failed,{S,init_per_group,E}}}}, + {?eh,tc_auto_skip,{S,{tc2,g},{failed,{S,init_per_group,E}}}}, + {?eh,test_stats,{2,0,{0,2}}}, + {?eh,tc_auto_skip,{S,{end_per_group,g},{failed,{S,init_per_group,E}}}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPG) when EPG=:=pre_end_per_group; EPG=:=post_end_per_group -> + S = curr_tc_SUITE, + FwFunc = + case EPG of + pre_end_per_group -> init_tc; + post_end_per_group -> end_tc + end, + E = {failed,{ct_framework,FwFunc,{test_case_failed,hahahahahah}}}, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},E}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(IPTC) when IPTC=:=pre_init_per_testcase; + IPTC=:=post_init_per_testcase -> + S = curr_tc_SUITE, + E = case IPTC of + pre_init_per_testcase -> + {failed,{ct_framework,init_tc,{test_case_failed,hahahahahah}}}; + post_init_per_testcase -> + {failed,{ct_framework,end_tc,{timetrap,3000}}} + end, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,init_per_suite}}, + {?eh,tc_done,{S,init_per_suite,ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,{auto_skipped,E}}}, + {?eh,test_stats,{0,0,{0,1}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{1,0,{0,1}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,{auto_skipped,E}}}, + {?eh,test_stats,{1,0,{0,2}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,2}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(EPTC) when EPTC=:=pre_end_per_testcase; EPTC=:=post_end_per_testcase-> + S = curr_tc_SUITE, + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + [{?eh,tc_start,{S,{init_per_group,g,[]}}}, + {?eh,tc_done,{S,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{S,tc1}}, + {?eh,tc_done,{S,tc1,ok}}, + {?eh,test_stats,{3,0,{0,0}}}, + {?eh,tc_start,{S,tc2}}, + {?eh,tc_done,{S,tc2,ok}}, + {?eh,test_stats,{4,0,{0,0}}}, + {?eh,tc_start,{S,{end_per_group,g,[]}}}, + {?eh,tc_done,{S,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{S,end_per_suite}}, + {?eh,tc_done,{S,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]. + +%% test events help functions +contains(List) -> + fun(Proplist) when is_list(Proplist) -> + contains(List,Proplist) + end. + +contains([{not_in_order,List}|T],Rest) -> + contains_parallel(List,Rest), + contains(T,Rest); +contains([{Ele,Pos}|T] = L,[H|T2]) -> + case element(Pos,H) of + Ele -> + contains(T,T2); + _ -> + contains(L,T2) + end; +contains([Ele|T],[{Ele,_}|T2])-> + contains(T,T2); +contains([Ele|T],[Ele|T2])-> + contains(T,T2); +contains(List,[_|T]) -> + contains(List,T); +contains([],_) -> + match. + +contains_parallel([Key | T], Elems) -> + contains([Key],Elems), + contains_parallel(T,Elems); +contains_parallel([],_Elems) -> + match. + +not_contains(List) -> + fun(Proplist) when is_list(Proplist) -> + [] = [Ele || {Ele,_} <- Proplist, + Test <- List, + Test =:= Ele] + end. diff --git a/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl b/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl new file mode 100644 index 0000000000..34c1568a87 --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE_data/ct_util_cth.erl @@ -0,0 +1,105 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(ct_util_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + erlang:system_time(second). + +init(Id, Opts) -> + {ok,ok}. + +pre_init_per_suite(Suite,Config,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Config, State}. + +post_init_per_suite(Suite,Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Return, State}. + +pre_end_per_suite(Suite,Config,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Config, State}. + +post_end_per_suite(Suite,Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite), + {Return, State}. + +pre_init_per_group(Suite, Group, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Config,State}. + +post_init_per_group(Suite, Group, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Return,State}. + +pre_end_per_group(Suite, Group, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Config,State}. + +post_end_per_group(Suite, Group, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,Group), + {Return,State}. + +pre_init_per_testcase(Suite, TC, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Config,State}. + +post_init_per_testcase(Suite, TC, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Return,State}. + +pre_end_per_testcase(Suite, TC, Config, State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Config,State}. + +post_end_per_testcase(Suite, TC, Config,Return,State) -> + maybe_sleep(?FUNCTION_NAME,Suite,TC), + {Return,State}. + +%%%----------------------------------------------------------------- +maybe_sleep(FuncName,Suite) -> + maybe_sleep(FuncName,Suite,undefined). +maybe_sleep(FuncName,Suite,GroupOrTC) -> + case ct:get_config(FuncName) of + {Suite,GroupOrTC,Fail} -> + fail(Fail); + {Suite,Fail} when GroupOrTC=:=undefined -> + fail(Fail); + _ -> + ok + end. + +fail({timeout,T}) -> + timer:sleep(T); +fail(kill) -> + spawn_link(fun() -> ct:fail(hahahahahah) end), + timer:sleep(10000). + diff --git a/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl b/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl new file mode 100644 index 0000000000..b48ba4d24e --- /dev/null +++ b/lib/common_test/test/ct_util_SUITE_data/curr_tc_SUITE.erl @@ -0,0 +1,59 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(curr_tc_SUITE). + +-suite_defaults([{timetrap, {seconds, 3}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +init_per_suite(Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + Config. + +end_per_suite(Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +init_per_group(_Group,Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + Config. + +end_per_group(_Group,Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +all() -> + [tc1,tc2,{group,g}]. + +groups() -> + [{g,[tc1,tc2]}]. + +%% Test cases starts here. +tc1(_Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. + +tc2(_Config) -> + [{?MODULE,?FUNCTION_NAME}] = ct_util:get_testdata(curr_tc), + ok. diff --git a/lib/common_test/test_server/configure.in b/lib/common_test/test_server/configure.in index 0511d126b4..e07bd4c2aa 100644 --- a/lib/common_test/test_server/configure.in +++ b/lib/common_test/test_server/configure.in @@ -459,11 +459,11 @@ dnl Freely inspired by AC_TRY_LINK. (Maybe better to create a dnl AC_LANG_JAVA instead...) AC_DEFUN(ERL_TRY_LINK_JAVA, [java_link='$JAVAC conftest.java 1>&AC_FD_CC' -changequote(�, �)dnl +changequote(, )dnl cat > conftest.java <<EOF -�$1� +$1 class conftest { public static void main(String[] args) { - �$2� + $2 ; return; }} EOF changequote([, ])dnl diff --git a/lib/common_test/test_server/ts_erl_config.erl b/lib/common_test/test_server/ts_erl_config.erl index 537628e39a..f3972bea4e 100644 --- a/lib/common_test/test_server/ts_erl_config.erl +++ b/lib/common_test/test_server/ts_erl_config.erl @@ -208,7 +208,11 @@ erl_interface(Vars,OsType) -> {filename:join(Dir, "lib"), filename:join([Dir, "src", "eidefs.mk"])}; {srctree, _Root, Target} -> - {filename:join([Dir, "obj", Target]), + Obj = case is_debug_build() of + true -> "obj.debug"; + false -> "obj" + end, + {filename:join([Dir, Obj, Target]), filename:join([Dir, "src", Target, "eidefs.mk"])} end} end, diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index fd5d4a57aa..14a3622a00 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.16.1 +COMMON_TEST_VSN = 1.17.1 |