diff options
Diffstat (limited to 'lib/common_test')
67 files changed, 2844 insertions, 972 deletions
diff --git a/lib/common_test/doc/src/basics_chapter.xml b/lib/common_test/doc/src/basics_chapter.xml index ff6ea6c557..12c6143a1a 100644 --- a/lib/common_test/doc/src/basics_chapter.xml +++ b/lib/common_test/doc/src/basics_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2003</year><year>2012</year> + <year>2003</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/book.xml b/lib/common_test/doc/src/book.xml index 2c48664b26..f3f28999d7 100644 --- a/lib/common_test/doc/src/book.xml +++ b/lib/common_test/doc/src/book.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE book SYSTEM "book.dtd"> <book xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index f446f8ac13..ea4522c40b 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> diff --git a/lib/common_test/doc/src/config_file_chapter.xml b/lib/common_test/doc/src/config_file_chapter.xml index 99e25faf27..d3068db17b 100644 --- a/lib/common_test/doc/src/config_file_chapter.xml +++ b/lib/common_test/doc/src/config_file_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2004</year><year>2012</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index b952df58f1..a215c8c2f3 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index b3e713c77f..859ff9df14 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -485,8 +485,9 @@ NewCTHState</name> <fsummary>Called after the CTH scope ends</fsummary> <type> - <v>TestcaseName = end_per_suite | init_per_group | - end_per_group | atom()</v> + <v>TestcaseName = end_per_suite | {init_per_group,GroupName} | + {end_per_group,GroupName} | atom()</v> + <v>GroupName = atom()</v> <v>Reason = {tc_auto_skip | tc_user_skip, term()}</v> <v>CTHState = NewCTHState = term()</v> </type> diff --git a/lib/common_test/doc/src/ct_master_chapter.xml b/lib/common_test/doc/src/ct_master_chapter.xml index 1d2d64a166..37a0805055 100644 --- a/lib/common_test/doc/src/ct_master_chapter.xml +++ b/lib/common_test/doc/src/ct_master_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2006</year><year>2012</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index d857b20d88..39259b092a 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE comref SYSTEM "comref.dtd"> <comref> diff --git a/lib/common_test/doc/src/dependencies_chapter.xml b/lib/common_test/doc/src/dependencies_chapter.xml index 8aad552285..e6c7025b12 100644 --- a/lib/common_test/doc/src/dependencies_chapter.xml +++ b/lib/common_test/doc/src/dependencies_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2006</year><year>2009</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/event_handler_chapter.xml b/lib/common_test/doc/src/event_handler_chapter.xml index 3cc21f28de..47d0ba59fb 100644 --- a/lib/common_test/doc/src/event_handler_chapter.xml +++ b/lib/common_test/doc/src/event_handler_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2006</year><year>2012</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -193,7 +193,8 @@ <p><c>GroupName = unknown | atom()</c>, name of the group (unknown if init- or end function times out).</p> <p><c>GroupProperties = list()</c>, list of execution properties for the group.</p> - <p><c>Result = ok | {skipped,SkipReason} | {failed,FailReason}</c>, the result.</p> + <p><c>Result = ok | {auto_skipped,SkipReason} | {skipped,SkipReason} | {failed,FailReason}</c>, + the result.</p> <marker id="skipreason"/> <p><c>SkipReason = {require_failed,RequireInfo} | {require_failed_in_suite0,RequireInfo} | @@ -228,7 +229,9 @@ <marker id="tc_auto_skip"></marker> <c>#event{name = tc_auto_skip, data = {Suite,Func,Reason}}</c> <p><c>Suite = atom()</c>, the name of the suite.</p> - <p><c>Func = atom()</c>, the name of the test case or configuration function.</p> + <p><c>Func = atom() | {end_per_group,GroupName}</c>, the name of the test case + or configuration function.</p> + <p><c>GroupName = atom()</c>, name of the group.</p> <p><c>Reason = {failed,FailReason} | {require_failed_in_suite0,RequireInfo}</c>, reason for auto skipping <c>Func</c>.</p> @@ -254,9 +257,11 @@ <item> <marker id="tc_user_skip"></marker> - <c>#event{name = tc_user_skip, data = {Suite,TestCase,Comment}}</c> - <p><c>Suite = atom()</c>, name of the suite.</p> - <p><c>TestCase = atom()</c>, name of the test case.</p> + <c>#event{name = tc_user_skip, data = {Suite,Func,Comment}}</c> + <p><c>Suite = atom()</c>, the name of the suite.</p> + <p><c>Func = atom() | {end_per_group,GroupName}</c>, the name of the test case + or configuration function.</p> + <p><c>GroupName = atom()</c>, name of the group.</p> <p><c>Comment = string()</c>, reason for skipping the test case.</p> <p>This event specifies that a test case has been skipped by the user. It is only ever received if the skip was declared in a test specification. diff --git a/lib/common_test/doc/src/example_chapter.xml b/lib/common_test/doc/src/example_chapter.xml index 2333f92989..36781d152c 100644 --- a/lib/common_test/doc/src/example_chapter.xml +++ b/lib/common_test/doc/src/example_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2003</year><year>2012</year> + <year>2003</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/fascicules.xml b/lib/common_test/doc/src/fascicules.xml index 38df4a1709..c4a28a699a 100644 --- a/lib/common_test/doc/src/fascicules.xml +++ b/lib/common_test/doc/src/fascicules.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE fascicules SYSTEM "fascicules.dtd"> <fascicules> diff --git a/lib/common_test/doc/src/getting_started_chapter.xml b/lib/common_test/doc/src/getting_started_chapter.xml index 0b42445540..445c731d01 100644 --- a/lib/common_test/doc/src/getting_started_chapter.xml +++ b/lib/common_test/doc/src/getting_started_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2007</year><year>2012</year> + <year>2007</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/install_chapter.xml b/lib/common_test/doc/src/install_chapter.xml index 4ef4e6de94..7f8c606324 100644 --- a/lib/common_test/doc/src/install_chapter.xml +++ b/lib/common_test/doc/src/install_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2007</year><year>2010</year> + <year>2007</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index fce5401f13..f10d5f85bf 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> @@ -32,6 +32,126 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.7.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Return values from group and testcase info functions are + now properly checked, and associated test cases are auto + skipped if a return value is invalid.</p> + <p> + Own Id: OTP-10631 Aux Id: kunagi-345 [256] </p> + </item> + <item> + <p>The way Common Test handles skipping of test cases has + been updated. In previous versions, returning + <c>{skip,Reason}</c> from a configuration function (such + as init_per_suite or init_per_group), resulted in all + affected test cases getting skipped with status + <c>auto_skipped</c>. This was inappropriate, since this + status is supposed to be used to inform that Common Test + has taken the initiative to skip something (e.g. a test + case group if init_per_group failed). Therefore, in this + version of Common Test, whenever the user skips a suite, + group, or individual test case (by means of a + configuration function or test specification term), the + affected test cases get the status <c>user_skipped</c> + instead.</p> <p>This update has meant a few changes that + may affect Common Test users in various ways: <list> + <item>The test results and statistics will be affected, + which is important to know when running regression tests + and comparing results to previous test runs.</item> + <item>Users that read or parse the textual log file + <c>suite.log</c> will notice that an auto skipped + function is now reported as <c>auto_skipped</c> rather + than <c>skipped</c> as before.</item> <item>When + <c>require</c> fails in an info function (such as suite/0 + or group/1), all affected configuration functions and + test cases are marked as <c>auto_skipped</c>.</item> + <item>If Common Test detects an error in the test suite + (such as e.g. an invalid all/0 function), all affected + configuration functions and test cases are marked as + <c>auto_skipped</c>.</item> <item>If a repeated test run + session reaches a deadline with <c>force_stop</c> + enabled, all remaining test cases are marked as + <c>auto_skipped</c> rather than <c>user_skipped</c> as + before.</item> <item>The event messages that Common Test + generates during test runs have been affected by this + update. For details see OTP-11524.</item> </list> </p> + <p> + Own Id: OTP-11305 Aux Id: OTP-11524 </p> + </item> + <item> + <p> + Returning {skip, Reason} from a pre_end_per_group/3 user + hook function would result in an exit in the Common Test + cth_log_redirect hook. This problem has been solved.</p> + <p> + Own Id: OTP-11409 Aux Id: seq12446 </p> + </item> + <item> + <p> + When the netconf server did not respond to the + close-session request, the call to + ct_netconfc:close_session/2 would hang forever waiting + for the netconf client to terminate. This has been + corrected. The client will now always terminate (and take + down the connection) if the close-session request times + out.</p> + <p> + Own Id: OTP-11478</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Fix cth_log_redirect.erl to fulfill gen_event behaviour. + Thanks to Roberto Aloi.</p> + <p> + Own Id: OTP-11401</p> + </item> + <item> + <p> + The first argument of the CT hook callback function + <c>on_tc_skip/3</c> has been modified. When this function + is called for <c>init_per_group</c> or + <c>end_per_group</c>, the value of the first argument is + now <c>{init_per_group,GroupName}</c> or + <c>{end_per_group,GroupName}</c>.</p> + <p> + Own Id: OTP-11523</p> + </item> + <item> + <p>The following modifications have been made to the + event messages that Common Test sends during test + execution: <list> <item>For the <c>tc_auto_skip</c> + event, the value of the <c>Func</c> element has changed + from <c>end_per_group</c> to + <c>{end_per_group,GroupName}</c>.</item> <item>When + <c>require</c> fails in an info function, such as suite/0 + or group/1, the init configuration function is now + reported as <c>auto_skipped</c> intead of <c>skipped</c>, + with the <c>tc_done</c> event.</item> <item>When + <c>require</c> fails in an info function because of a + configuration name already in use, the <c>tc_done</c> + event now reports the error with a tuple (of size 2) + tagged <c>failed</c> instead of <c>skipped</c>.</item> + </list> Please see the Event Handling chapter in the + Common Test User's Guide for reference. </p> + <p> + Own Id: OTP-11524 Aux Id: OTP-11305 </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.7.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/notes_history.xml b/lib/common_test/doc/src/notes_history.xml index e4085d5431..34ac4fd15b 100644 --- a/lib/common_test/doc/src/notes_history.xml +++ b/lib/common_test/doc/src/notes_history.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2004</year><year>2009</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/part.xml b/lib/common_test/doc/src/part.xml index a74185221d..fe73918975 100644 --- a/lib/common_test/doc/src/part.xml +++ b/lib/common_test/doc/src/part.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2011</year> + <year>2003</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/part_notes.xml b/lib/common_test/doc/src/part_notes.xml index 506677a7a5..13ca499196 100644 --- a/lib/common_test/doc/src/part_notes.xml +++ b/lib/common_test/doc/src/part_notes.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> <part xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2004</year><year>2009</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/part_notes_history.xml b/lib/common_test/doc/src/part_notes_history.xml index e658810e7a..f995a70b7f 100644 --- a/lib/common_test/doc/src/part_notes_history.xml +++ b/lib/common_test/doc/src/part_notes_history.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> <part> <header> <copyright> - <year>2004</year><year>2009</year> + <year>2004</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index 6fede88434..2f5c892e60 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE application SYSTEM "application.dtd"> <application xmlns:xi="http://www.w3.org/2001/XInclude"> <header> <copyright> - <year>2003</year><year>2012</year> + <year>2003</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 6b37e183dd..a4a77ee400 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> @@ -205,17 +205,17 @@ <p>The <c>ct_run</c> program sets the exit status before shutting down. The following values are defined:</p> <list> - <item><c>0</c> indicates a successful testrun, i.e. one without failed or auto-skipped test cases.</item> - <item><c>1</c> indicates that one or more test cases have failed, or have been auto-skipped.</item> + <item><c>0</c> indicates a successful testrun, i.e. one without failed or auto skipped test cases.</item> + <item><c>1</c> indicates that one or more test cases have failed, or have been auto skipped.</item> <item><c>2</c> indicates that the test execution has failed because of e.g. compilation errors, an illegal return value from an info function, etc.</item> </list> - <p>If auto-skipped test cases should not affect the exit status, you may change the default + <p>If auto skipped test cases should not affect the exit status, you may change the default behaviour using start flag:</p> <pre>-exit_status ignore_config</pre> <p>For more information about the <c>ct_run</c> program, see the - <seealso marker="ct_run#top">Reference Manual</seealso> and the + <seealso marker="ct_run">Reference Manual</seealso> and the <seealso marker="install_chapter#general">Installation</seealso> chapter. </p> </section> diff --git a/lib/common_test/doc/src/test_structure_chapter.xml b/lib/common_test/doc/src/test_structure_chapter.xml index b9ca59135d..3c9e159436 100644 --- a/lib/common_test/doc/src/test_structure_chapter.xml +++ b/lib/common_test/doc/src/test_structure_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2006</year><year>2010</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/why_test_chapter.xml b/lib/common_test/doc/src/why_test_chapter.xml index 95ff614f87..db94c6a6cb 100644 --- a/lib/common_test/doc/src/why_test_chapter.xml +++ b/lib/common_test/doc/src/why_test_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2003</year><year>2009</year> + <year>2003</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index a33b22ac39..055fb90641 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> diff --git a/lib/common_test/src/common_test.appup.src b/lib/common_test/src/common_test.appup.src index 0fbe5f23f7..4dfd9f1b0d 100644 --- a/lib/common_test/src/common_test.appup.src +++ b/lib/common_test/src/common_test.appup.src @@ -1 +1,21 @@ -{"%VSN%",[],[]}.
\ No newline at end of file +%% -*- erlang -*- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +{"%VSN%", + [{<<".*">>,[{restart_application, common_test}]}], + [{<<".*">>,[{restart_application, common_test}]}] +}. diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index ac08a3e0ad..d733df27dc 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -30,80 +30,88 @@ handle_event/2, handle_call/2, handle_info/2, terminate/2]). --record(state, {group_leader,logs=[]}). +-record(state, {logs=[], default_gl}). -define(WIDTH,80). %%%----------------------------------------------------------------- %%% Callbacks -init({GL,Logs}) -> - open_files(Logs,#state{group_leader=GL}). +init({GL,ConnLogs}) -> + open_files(GL,ConnLogs,#state{default_gl=GL}). -open_files([{ConnMod,{LogType,Logs}}|T],State) -> - case do_open_files(Logs,[]) of +open_files(GL,[{ConnMod,{LogType,LogFiles}}|T],State=#state{logs=Logs}) -> + case do_open_files(LogFiles,[]) of {ok,Fds} -> - open_files(T,State#state{logs=[{ConnMod,{LogType,Fds}} | - State#state.logs]}); + ConnInfo = proplists:get_value(GL,Logs,[]), + Logs1 = [{GL,[{ConnMod,{LogType,Fds}}|ConnInfo]} | + proplists:delete(GL,Logs)], + open_files(GL,T,State#state{logs=Logs1}); Error -> Error end; -open_files([],State) -> +open_files(_GL,[],State) -> {ok,State}. - -do_open_files([{Tag,File}|Logs],Acc) -> - case file:open(File, [write,{encoding,utf8}]) of +do_open_files([{Tag,File}|LogFiles],Acc) -> + case file:open(File, [write,append,{encoding,utf8}]) of {ok,Fd} -> - do_open_files(Logs,[{Tag,Fd}|Acc]); + do_open_files(LogFiles,[{Tag,Fd}|Acc]); {error,Reason} -> {error,{could_not_open_log,File,Reason}} end; do_open_files([],Acc) -> {ok,lists:reverse(Acc)}. +handle_event({info_report,_,{From,update,{GL,ConnLogs}}}, + State) when node(GL) == node() -> + Result = open_files(GL,ConnLogs,State), + From ! {updated,GL}, + Result; handle_event({_Type, GL, _Msg}, State) when node(GL) /= node() -> {ok, State}; -handle_event({_Type,_GL,{Pid,{ct_connection,Action,ConnName},Report}},State) -> - %% NOTE: if the format of this event is changed - %% ({ct_connection,Action,ConnName}) then remember to change - %% test_server_h:report_receiver as well!!! - Info = conn_info(Pid,#conn_log{name=ConnName,action=Action}), - write_report(now(),Info,Report,State), +handle_event({_Type,GL,{Pid,{ct_connection,Mod,Action,ConnName},Report}}, + State) -> + Info = conn_info(Pid,#conn_log{name=ConnName,action=Action,module=Mod}), + write_report(now(),Info,Report,GL,State), {ok, State}; -handle_event({_Type,_GL,{Pid,Info=#conn_log{},Report}},State) -> - %% NOTE: if the format of this event is changed - %% (Info=#conn_log{}) then remember to change - %% test_server_h:report_receiver as well!!! - write_report(now(),conn_info(Pid,Info),Report,State), +handle_event({_Type,GL,{Pid,Info=#conn_log{},Report}}, State) -> + write_report(now(),conn_info(Pid,Info),Report,GL,State), {ok, State}; -handle_event({error_report,_,{Pid,_,[{ct_connection,ConnName}|R]}},State) -> +handle_event({error_report,GL,{Pid,_,[{ct_connection,ConnName}|R]}}, State) -> %% Error reports from connection - write_error(now(),conn_info(Pid,#conn_log{name=ConnName}),R,State), + write_error(now(),conn_info(Pid,#conn_log{name=ConnName}),R,GL,State), {ok, State}; -handle_event(_, State) -> +handle_event(_What, State) -> {ok, State}. -handle_info(_, State) -> +handle_info(_What, State) -> {ok, State}. handle_call(_Query, State) -> {ok, {error, bad_query}, State}. terminate(_,#state{logs=Logs}) -> - [file:close(Fd) || {_,{_,Fds}} <- Logs, {_,Fd} <- Fds], + lists:foreach( + fun({_GL,ConnLogs}) -> + [file:close(Fd) || {_,{_,Fds}} <- ConnLogs, {_,Fd} <- Fds] + end, Logs), ok. %%%----------------------------------------------------------------- %%% Writing reports -write_report(Time,#conn_log{module=ConnMod}=Info,Data,State) -> - {LogType,Fd} = get_log(Info,State), +write_report(_Time,#conn_log{header=false,module=ConnMod}=Info,Data,GL,State) -> + {LogType,Fd} = get_log(Info,GL,State), + io:format(Fd,"~n~ts",[format_data(ConnMod,LogType,Data)]); + +write_report(Time,#conn_log{module=ConnMod}=Info,Data,GL,State) -> + {LogType,Fd} = get_log(Info,GL,State), io:format(Fd,"~n~ts~ts~ts",[format_head(ConnMod,LogType,Time), format_title(LogType,Info), format_data(ConnMod,LogType,Data)]). -write_error(Time,#conn_log{module=ConnMod}=Info,Report,State) -> - case get_log(Info,State) of +write_error(Time,#conn_log{module=ConnMod}=Info,Report,GL,State) -> + case get_log(Info,GL,State) of {html,_} -> %% The error will anyway be written in the html log by the %% sasl error handler, so don't write it again. @@ -115,14 +123,19 @@ write_error(Time,#conn_log{module=ConnMod}=Info,Report,State) -> format_error(LogType,Report)]) end. -get_log(Info,State) -> - case proplists:get_value(Info#conn_log.module,State#state.logs) of - {html,_} -> - {html,State#state.group_leader}; - {LogType,Fds} -> - {LogType,get_fd(Info,Fds)}; +get_log(Info,GL,State) -> + case proplists:get_value(GL,State#state.logs) of undefined -> - {html,State#state.group_leader} + {html,State#state.default_gl}; + ConnLogs -> + case proplists:get_value(Info#conn_log.module,ConnLogs) of + {html,_} -> + {html,GL}; + {LogType,Fds} -> + {LogType,get_fd(Info,Fds)}; + undefined -> + {html,GL} + end end. get_fd(#conn_log{name=undefined},Fds) -> @@ -191,17 +204,27 @@ pretty_head({{{Y,Mo,D},{H,Mi,S}},MicroS},ConnMod,Text0) -> micro2milli(MicroS)]). pretty_title(#conn_log{client=Client}=Info) -> - io_lib:format("= Client ~w ~s Server ~ts ", - [Client,actionstr(Info),serverstr(Info)]). + case actionstr(Info) of + {no_server,Action} -> + io_lib:format("= Client ~w ~s ",[Client,Action]); + Action -> + io_lib:format("= Client ~w ~s ~ts ",[Client,Action, + serverstr(Info)]) + end. actionstr(#conn_log{action=send}) -> "----->"; +actionstr(#conn_log{action=cmd}) -> "----->"; actionstr(#conn_log{action=recv}) -> "<-----"; actionstr(#conn_log{action=open}) -> "opened session to"; actionstr(#conn_log{action=close}) -> "closed session to"; actionstr(_) -> "<---->". +serverstr(#conn_log{name=undefined,address={undefined,_}}) -> + io_lib:format("server",[]); serverstr(#conn_log{name=undefined,address=Address}) -> io_lib:format("~p",[Address]); +serverstr(#conn_log{name=Alias,address={undefined,_}}) -> + io_lib:format("~w",[Alias]); serverstr(#conn_log{name=Alias,address=Address}) -> io_lib:format("~w(~p)",[Alias,Address]). diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 276f902b05..54510a657a 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -69,13 +69,13 @@ init_tc(Mod,Func,Config) -> andalso Func=/=end_per_group andalso ct_util:get_testdata(skip_rest) of true -> - {skip,"Repeated test stopped by force_stop option"}; + {auto_skip,"Repeated test stopped by force_stop option"}; _ -> case ct_util:get_testdata(curr_tc) of {Suite,{suite0_failed,{require,Reason}}} -> - {skip,{require_failed_in_suite0,Reason}}; + {auto_skip,{require_failed_in_suite0,Reason}}; {Suite,{suite0_failed,_}=Failure} -> - {skip,Failure}; + {fail,Failure}; _ -> ct_util:update_testdata(curr_tc, fun(undefined) -> @@ -103,7 +103,7 @@ init_tc(Mod,Func,Config) -> end, init_tc1(Mod,Suite,Func,Config); {failed,Seq,BadFunc} -> - {skip,{sequence_failed,Seq,BadFunc}} + {auto_skip,{sequence_failed,Seq,BadFunc}} end end end. @@ -115,9 +115,9 @@ init_tc1(?MODULE,_,error_in_suite,[Config0]) when is_list(Config0) -> data={?MODULE,error_in_suite}}), case ?val(error, Config0) of undefined -> - {skip,"unknown_error_in_suite"}; + {fail,"unknown_error_in_suite"}; Reason -> - {skip,Reason} + {fail,Reason} end; init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) -> @@ -159,22 +159,28 @@ init_tc1(Mod,Suite,Func,[Config0]) when is_list(Config0) -> true -> ct_config:delete_default_config(testcase) end, + Initialize = fun() -> + ct_logs:init_tc(false), + ct_event:notify(#event{name=tc_start, + node=node(), + data={Mod,FuncSpec}}) + end, case add_defaults(Mod,Func,AllGroups) of Error = {suite0_failed,_} -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}), + Initialize(), ct_util:set_testdata({curr_tc,{Suite,Error}}), {error,Error}; + Error = {group0_failed,_} -> + Initialize(), + {auto_skip,Error}; + Error = {testcase0_failed,_} -> + Initialize(), + {auto_skip,Error}; {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}), - {skip,Reason}; + Initialize(), + {fail,Reason}; _ -> init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) end @@ -222,11 +228,11 @@ init_tc2(Mod,Suite,Func,SuiteInfo,MergeResult,Config) -> {suite0_failed,Reason} -> ct_util:set_testdata({curr_tc,{Mod,{suite0_failed, {require,Reason}}}}), - {skip,{require_failed_in_suite0,Reason}}; + {auto_skip,{require_failed_in_suite0,Reason}}; {error,Reason} -> {auto_skip,{require_failed,Reason}}; {'EXIT',Reason} -> - {auto_skip,Reason}; + {fail,Reason}; {ok,PostInitHook,Config1} -> case get('$test_server_framework_test') of undefined -> @@ -272,6 +278,8 @@ add_defaults(Mod,Func, GroupPath) -> SuiteInfo = merge_with_suite_defaults(Suite,[]), SuiteInfoNoCTH = [I || I <- SuiteInfo, element(1,I) =/= ct_hooks], case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of + Error = {group0_failed,_} -> Error; + Error = {testcase0_failed,_} -> Error; Error = {error,_} -> {SuiteInfo,Error}; MergedInfo -> {SuiteInfo,MergedInfo} end; @@ -292,13 +300,16 @@ add_defaults(Mod,Func, GroupPath) -> element(1,I) =/= ct_hooks], case add_defaults1(Mod,Func, GroupPath, SuiteInfoNoCTH) of + Error = {group0_failed,_} -> Error; + Error = {testcase0_failed,_} -> Error; Error = {error,_} -> {SuiteInfo1,Error}; MergedInfo -> {SuiteInfo1,MergedInfo} end; false -> ErrStr = io_lib:format("~n*** ERROR *** " "Invalid return value from " - "~w:suite/0: ~p~n", [Suite,SuiteInfo]), + "~w:suite/0: ~p~n", + [Suite,SuiteInfo]), io:format(ErrStr, []), io:format(user, ErrStr, []), {suite0_failed,bad_return_value} @@ -318,36 +329,69 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) -> %% [LevelXGroupInfo, LevelX-1GroupInfo, ..., TopLevelGroupInfo] GroupPathInfo = lists:map(fun(GroupProps) -> - Name = ?val(name, GroupProps), - case catch Suite:group(Name) of - GrInfo when is_list(GrInfo) -> GrInfo; - _ -> [] + case ?val(name, GroupProps) of + undefined -> + []; + Name -> + case catch Suite:group(Name) of + GrInfo when is_list(GrInfo) -> GrInfo; + {'EXIT',{undef,_}} -> []; + BadGr0 -> {error,BadGr0,Name} + end end end, GroupPath), - Args = if Func == init_per_group ; Func == end_per_group -> - [?val(name, hd(GroupPath))]; - true -> - [] - end, - TestCaseInfo = - case catch apply(Mod,Func,Args) of - TCInfo when is_list(TCInfo) -> TCInfo; - _ -> [] - end, - %% let test case info (also for all config funcs) override group info, - %% and lower level group info override higher level info - TCAndGroupInfo = [TestCaseInfo | remove_info_in_prev(TestCaseInfo, - GroupPathInfo)], - %% find and save require terms found in suite info - SuiteReqs = - [SDDef || SDDef <- SuiteInfo, - ((require == element(1,SDDef)) or - (default_config == element(1,SDDef)))], - case check_for_clashes(TestCaseInfo, GroupPathInfo, SuiteReqs) of - [] -> - add_defaults2(Mod,Func, TCAndGroupInfo,SuiteInfo,SuiteReqs); - Clashes -> - {error,{config_name_already_in_use,Clashes}} + case lists:keysearch(error, 1, GroupPathInfo) of + {value,{error,BadGr0Val,GrName}} -> + Gr0ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:group(~w): ~p~n", + [Mod,GrName,BadGr0Val]), + io:format(Gr0ErrStr, []), + io:format(user, Gr0ErrStr, []), + {group0_failed,bad_return_value}; + _ -> + Args = if Func == init_per_group ; Func == end_per_group -> + [?val(name, hd(GroupPath))]; + true -> + [] + end, + TestCaseInfo = + case catch apply(Mod,Func,Args) of + TCInfo when is_list(TCInfo) -> TCInfo; + {'EXIT',{undef,_}} -> []; + BadTC0 -> {error,BadTC0} + end, + + case TestCaseInfo of + {error,BadTC0Val} -> + TC0ErrStr = io_lib:format("~n*** ERROR *** " + "Invalid return value from " + "~w:~w/0: ~p~n", + [Mod,Func,BadTC0Val]), + io:format(TC0ErrStr, []), + io:format(user, TC0ErrStr, []), + {testcase0_failed,bad_return_value}; + _ -> + %% let test case info (also for all config funcs) override + %% group info, and lower level group info override higher + %% level info + TCAndGroupInfo = + [TestCaseInfo | remove_info_in_prev(TestCaseInfo, + GroupPathInfo)], + %% find and save require terms found in suite info + SuiteReqs = + [SDDef || SDDef <- SuiteInfo, + ((require == element(1,SDDef)) + or (default_config == element(1,SDDef)))], + case check_for_clashes(TestCaseInfo, GroupPathInfo, + SuiteReqs) of + [] -> + add_defaults2(Mod,Func, TCAndGroupInfo, + SuiteInfo,SuiteReqs); + Clashes -> + {error,{config_name_already_in_use,Clashes}} + end + end end. get_suite_name(?MODULE, [Cfg|_]) when is_list(Cfg), Cfg /= [] -> @@ -621,31 +665,34 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> _ -> Func end, - case get('$test_server_framework_test') of - undefined -> - {FinalResult,FinalNotify} = - case ct_hooks:end_tc( - Suite, FuncSpec, Args, Result, Return) of - '$ct_no_change' -> - {ok,Result}; - FinalResult1 -> - {FinalResult1,FinalResult1} - end, - %% send sync notification so that event handlers may print - %% in the log file before it gets closed - ct_event:sync_notify(#event{name=tc_done, - node=node(), - data={Mod,FuncSpec, - tag_cth(FinalNotify)}}); - Fun -> - %% send sync notification so that event handlers may print - %% in the log file before it gets closed - ct_event:sync_notify(#event{name=tc_done, - node=node(), - data={Mod,FuncSpec,tag(Result)}}), - FinalResult = Fun(end_tc, Return) - end, - + {Result1,FinalNotify} = + case ct_hooks:end_tc( + Suite, FuncSpec, Args, Result, Return) of + '$ct_no_change' -> + {ok,Result}; + HookResult -> + {HookResult,HookResult} + end, + FinalResult = + case get('$test_server_framework_test') of + undefined -> + %% send sync notification so that event handlers may print + %% in the log file before it gets closed + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={Mod,FuncSpec, + tag_cth(FinalNotify)}}), + Result1; + Fun -> + %% send sync notification so that event handlers may print + %% in the log file before it gets closed + ct_event:sync_notify(#event{name=tc_done, + node=node(), + data={Mod,FuncSpec, + tag(FinalNotify)}}), + Fun(end_tc, Return) + end, + case FuncSpec of {_,GroupName,_Props} -> if Func == end_per_group -> @@ -683,12 +730,17 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> (undefined) -> undefined; (Unexpected) -> - exit({error,{reset_curr_tc,{Mod,Func},Unexpected}}) + {error,{reset_curr_tc,{Mod,Func},Unexpected}} end, - ct_util:update_testdata(curr_tc,ClearCurrTC), + case ct_util:update_testdata(curr_tc, ClearCurrTC) of + {error,_} = ClearError -> + exit(ClearError); + _ -> + ok + end, case FinalResult of - {skip,{sequence_failed,_,_}} -> + {auto_skip,{sequence_failed,_,_}} -> %% ct_logs:init_tc is never called for a skipped test case %% in a failing sequence, so neither should end_tc ok; @@ -711,8 +763,13 @@ end_tc(Mod,Func,TCPid,Result,Args,Return) -> %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | %% {'EXIT',Reason} | Other (ignored return value, e.g. 'ok') -tag({STag,Reason}) when STag == skip; STag == skipped -> - {skipped,Reason}; +tag({STag,Reason}) when STag == skip; STag == skipped -> + case Reason of + {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason}; + _ -> {skipped,Reason} + end; +tag({auto_skip,Reason}) -> + {auto_skipped,Reason}; tag(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; ETag == timetrap_timeout; ETag == testcase_aborted -> @@ -722,13 +779,20 @@ tag(E = testcase_aborted_or_killed) -> tag(Other) -> Other. +tag_cth({skipped,Reason={failed,{_,init_per_testcase,_}}}) -> + {auto_skipped,Reason}; tag_cth({STag,Reason}) when STag == skip; STag == skipped -> - {skipped,Reason}; -tag_cth({fail, Reason}) -> - {failed, {error,Reason}}; + case Reason of + {failed,{_,init_per_testcase,_}} -> {auto_skipped,Reason}; + _ -> {skipped,Reason} + end; +tag_cth({auto_skip,Reason}) -> + {auto_skipped,Reason}; +tag_cth({fail,Reason}) -> + {failed,{error,Reason}}; tag_cth(E = {ETag,_}) when ETag == error; ETag == 'EXIT'; - ETag == timetrap_timeout; - ETag == testcase_aborted -> + ETag == timetrap_timeout; + ETag == testcase_aborted -> {failed,E}; tag_cth(E = testcase_aborted_or_killed) -> {failed,E}; @@ -1229,6 +1293,8 @@ report(What,Data) -> ct_hooks:on_tc_skip(tc_auto_skip, Data); {skipped,_} -> ct_hooks:on_tc_skip(tc_user_skip, Data); + {auto_skipped,_} -> + ct_hooks:on_tc_skip(tc_auto_skip, Data); _Else -> ok end, @@ -1253,31 +1319,54 @@ report(What,Data) -> add_to_stats(auto_skipped); {_,{skipped,_}} -> add_to_stats(user_skipped); + {_,{auto_skipped,_}} -> + add_to_stats(auto_skipped); {_,{SkipOrFail,_Reason}} -> add_to_stats(SkipOrFail) end; - tc_user_skip -> - %% test case specified as skipped in testspec - %% Data = {Suite,Case,Comment} + tc_user_skip -> + %% test case or config function specified as skipped in testspec, + %% or init config func for suite/group has returned {skip,Reason} + %% Data = {Suite,Case,Comment} | + %% {Suite,{GroupConfigFunc,GroupName},Comment} + {Func,Data1} = case Data of + {Suite,{ConfigFunc,undefined},Cmt} -> + {ConfigFunc,{Suite,ConfigFunc,Cmt}}; + {_,{ConfigFunc,_},_} -> {ConfigFunc,Data}; + {_,Case,_} -> {Case,Data} + end, + ct_event:sync_notify(#event{name=tc_user_skip, node=node(), - data=Data}), - ct_hooks:on_tc_skip(What, Data), - add_to_stats(user_skipped); - tc_auto_skip -> - %% test case skipped because of error in init_per_suite - %% Data = {Suite,Case,Comment} - - {_Suite,Case,_Result} = Data, + data=Data1}), + ct_hooks:on_tc_skip(What, Data1), + if Func /= init_per_suite, Func /= init_per_group, + Func /= end_per_suite, Func /= end_per_group -> + add_to_stats(user_skipped); + true -> + ok + end; + tc_auto_skip -> + %% test case skipped because of error in config function, or + %% config function skipped because of error in info function + %% Data = {Suite,Case,Comment} | + %% {Suite,{GroupConfigFunc,GroupName},Comment} + {Func,Data1} = case Data of + {Suite,{ConfigFunc,undefined},Cmt} -> + {ConfigFunc,{Suite,ConfigFunc,Cmt}}; + {_,{ConfigFunc,_},_} -> {ConfigFunc,Data}; + {_,Case,_} -> {Case,Data} + end, %% this test case does not have a log, so printouts %% from event handlers should end up in the main log ct_event:sync_notify(#event{name=tc_auto_skip, node=node(), - data=Data}), - ct_hooks:on_tc_skip(What, Data), - if Case /= end_per_suite, - Case /= end_per_group -> + data=Data1}), + ct_hooks:on_tc_skip(What, Data1), + + if Func /= end_per_suite, + Func /= end_per_group -> add_to_stats(auto_skipped); true -> ok diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index a5b736136f..078d6b1a44 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -29,6 +29,13 @@ -export([start/4, stop/1, get_conn_pid/1]). -export([call/2, call/3, return/2, do_within_time/2]). +%%---------------------------------------------------------------------- +%% Exported types +%%---------------------------------------------------------------------- +-export_type([server_id/0, + target_name/0, + key_or_name/0]). + -ifdef(debug). -define(dbg,true). -else. @@ -47,6 +54,18 @@ cb_state, ct_util_server}). +%%------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------ +-type server_id() :: atom(). +%% A `ServerId' which exists in a configuration file. +-type target_name() :: atom(). +%% A name which is associated to a `server_id()' via a +%% `require' statement or a call to {@link ct:require/2} in the +%% test suite. +-type key_or_name() :: server_id() | target_name(). + + %%%----------------------------------------------------------------- %%% @spec start(Address,InitData,CallbackMod,Opts) -> %%% {ok,Handle} | {error,Reason} diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 1a6e4d31a8..a7fb45a4e4 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -1249,7 +1249,8 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, integer_to_list(NotBuilt),"</a></td>\n"] end, FailStr = - if Fail > 0 -> + if (Fail > 0) or (NotBuilt > 0) or + ((Success+Fail+UserSkip+AutoSkip) == 0) -> ["<font color=\"red\">", integer_to_list(Fail),"</font>"]; true -> @@ -1904,7 +1905,8 @@ runentry(Dir, undefined, _) -> runentry(Dir, Totals={Node,Label,Logs, {TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}}, Index) -> TotFailStr = - if TotFail > 0 -> + if (TotFail > 0) or (NotBuilt > 0) or + ((TotSucc+TotFail+UserSkip+AutoSkip) == 0) -> ["<font color=\"red\">", integer_to_list(TotFail),"</font>"]; true -> diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 7f10e1db09..35920ec1dc 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -212,11 +212,7 @@ %%---------------------------------------------------------------------- %% Exported types %%---------------------------------------------------------------------- --export_type([hook_options/0, - conn_mod/0, - log_type/0, - key_or_name/0, - notification/0]). +-export_type([notification/0]). %%---------------------------------------------------------------------- %% Internal exports @@ -292,19 +288,11 @@ %%---------------------------------------------------------------------- %% Type declarations %%---------------------------------------------------------------------- --type client() :: handle() | server_id() | target_name(). +-type client() :: handle() | ct_gen_conn:server_id() | ct_gen_conn:target_name(). -type handle() :: term(). %% An opaque reference for a connection (netconf session). See {@link %% ct} for more information. --type server_id() :: atom(). -%% A `ServerId' which exists in a configuration file. --type target_name() :: atom(). -%% A name which is associated to a `server_id()' via a -%% `require' statement or a call to {@link ct:require/2} in the -%% test suite. --type key_or_name() :: server_id() | target_name(). - -type options() :: [option()]. %% Options used for setting up ssh connection to a netconf server. @@ -326,14 +314,7 @@ %% See XML Schema for Event Notifications found in RFC5277 for further %% detail about the data format for the string values. --type hook_options() :: [hook_option()]. -%% Options that can be given to `cth_conn_log' in the `ct_hook' statement. --type hook_option() :: {log_type,log_type()} | - {hosts,[key_or_name()]}. --type log_type() :: raw | pretty | html | silent. %-type error_handler() :: module(). --type conn_mod() :: ct_netconfc. - -type error_reason() :: term(). -type simple_xml() :: {xml_tag(), xml_attributes(), xml_content()} | @@ -384,7 +365,7 @@ open(Options) -> %%---------------------------------------------------------------------- -spec open(KeyOrName, ExtraOptions) -> Result when - KeyOrName :: key_or_name(), + KeyOrName :: ct_gen_conn:key_or_name(), ExtraOptions :: options(), Result :: {ok,handle()} | {error,error_reason()}. %% @doc Open a named netconf session and exchange `hello' messages. @@ -461,7 +442,7 @@ only_open(Options) -> %%---------------------------------------------------------------------- -spec only_open(KeyOrName,ExtraOptions) -> Result when - KeyOrName :: key_or_name(), + KeyOrName :: ct_gen_conn:key_or_name(), ExtraOptions :: options(), Result :: {ok,handle()} | {error,error_reason()}. %% @doc Open a name netconf session, but don't send `hello'. @@ -1129,10 +1110,14 @@ handle_msg({Ref,timeout}, ct_gen_conn:return(Caller,{error,{hello_session_failed,timeout}}), {stop,State#state{hello_status={error,timeout}}}; handle_msg({Ref,timeout},#state{pending=Pending} = State) -> - {value,#pending{caller=Caller},Pending1} = + {value,#pending{op=Op,caller=Caller},Pending1} = lists:keytake(Ref,#pending.ref,Pending), ct_gen_conn:return(Caller,{error,timeout}), - {noreply,State#state{pending=Pending1}}. + R = case Op of + close_session -> stop; + _ -> noreply + end, + {R,State#state{pending=Pending1}}. %% @private %% Called by ct_util_server to close registered connections before terminate. diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index 4092d33bc0..b4d82a53cf 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -17,51 +17,147 @@ %% %CopyrightEnd% %% -%%% @doc Common Test specific layer on top of telnet client ct_telnet_client.erl -%%% -%%% <p>Use this module to set up telnet connections, send commands and -%%% perform string matching on the result. -%%% See the <c>unix_telnet</c> manual page for information about how to use -%%% ct_telnet, and configure connections, specifically for unix hosts.</p> -%%% <p>The following default values are defined in ct_telnet:</p> -%%% <pre> -%%% Connection timeout = 10 sec (time to wait for connection) -%%% Command timeout = 10 sec (time to wait for a command to return) -%%% Max no of reconnection attempts = 3 -%%% Reconnection interval = 5 sek (time to wait in between reconnection attempts) -%%% Keep alive = true (will send NOP to the server every 10 sec if connection is idle)</pre> -%%% <p>These parameters can be altered by the user with the following -%%% configuration term:</p> -%%% <pre> -%%% {telnet_settings, [{connect_timeout,Millisec}, -%%% {command_timeout,Millisec}, -%%% {reconnection_attempts,N}, -%%% {reconnection_interval,Millisec}, -%%% {keep_alive,Bool}]}.</pre> -%%% <p><code>Millisec = integer(), N = integer()</code></p> -%%% <p>Enter the <code>telnet_settings</code> term in a configuration -%%% file included in the test and ct_telnet will retrieve the information -%%% automatically. Note that <c>keep_alive</c> may be specified per connection if -%%% required. See <c>unix_telnet</c> for details.</p></doc> - -%%% @type connection_type() = telnet | ts1 | ts2 - -%%% @type connection() = handle() | -%%% {ct:target_name(),connection_type()} | ct:target_name() - -%%% @type handle() = ct_gen_conn:handle(). Handle for a -%%% specific telnet connection. - -%%% @type prompt_regexp() = string(). A regular expression which -%%% matches all possible prompts for a specific type of target. The -%%% regexp must not have any groups i.e. when matching, re:run/3 shall -%%% return a list with one single element. -%%% -%%% @see unix_telnet +%% @doc Common Test specific layer on top of telnet client `ct_telnet_client.erl' +%% +%% <p>Use this module to set up telnet connections, send commands and +%% perform string matching on the result. +%% See the `unix_telnet' manual page for information about how to use +%% `ct_telnet', and configure connections, specifically for unix hosts.</p> +%% <p>The following default values are defined in `ct_telnet':</p> +%% <pre> +%% Connection timeout = 10 sec (time to wait for connection) +%% Command timeout = 10 sec (time to wait for a command to return) +%% Max no of reconnection attempts = 3 +%% Reconnection interval = 5 sek (time to wait in between reconnection attempts) +%% Keep alive = true (will send NOP to the server every 10 sec if connection is idle)</pre> +%% <p>These parameters can be altered by the user with the following +%% configuration term:</p> +%% <pre> +%% {telnet_settings, [{connect_timeout,Millisec}, +%% {command_timeout,Millisec}, +%% {reconnection_attempts,N}, +%% {reconnection_interval,Millisec}, +%% {keep_alive,Bool}]}.</pre> +%% <p><code>Millisec = integer(), N = integer()</code></p> +%% <p>Enter the <code>telnet_settings</code> term in a configuration +%% file included in the test and ct_telnet will retrieve the information +%% automatically. Note that `keep_alive' may be specified per connection if +%% required. See `unix_telnet' for details.</p> +%% +%% == Logging == +%% +%% `ct_telnet' can be configured to uses the `error_logger' for logging telnet +%% traffic. A special purpose error handler is implemented in +%% `ct_conn_log_h'. To use this error handler, add the `cth_conn_log' +%% hook in your test suite, e.g: +%% +%% +%% ``` +%% suite() -> +%% [{ct_hooks, [{cth_conn_log, [{conn_mod(),hook_options()}]}]}]. +%%''' +%% +%% `conn_mod()' is the name of the common_test module implementing +%% the connection protocol, i.e. `ct_telnet'. +%% +%% The hook option `log_type' specifies the type of logging: +%% +%% <dl> +%% <dt>`raw'</dt> +%% <dd>The sent and received telnet data is logged to a separate +%% text file as is, without any formatting. A link to the file is +%% added to the test case HTML log.</dd> +%% +%% <dt>`html (default)'</dt> +%% <dd>The sent and received telnet traffic is pretty printed +%% directly in the test case HTML log.</dd> +%% +%% <dt>`silent'</dt> +%% <dd>Telnet traffic is not logged.</dd> +%% </dl> +%% +%% By default, all telnet traffic is logged in one single log +%% file. However, it is possible to have different connections logged +%% in separate files. To do this, use the hook option `hosts' and +%% list the names of the servers/connections that will be used in the +%% suite. Note that the connections must be named for this to work +%% (see the `open' function below). +%% +%% The `hosts' option has no effect if `log_type' is set to `html' or +%% `silent'. +%% +%% The hook options can also be specified in a configuration file with +%% the configuration variable `ct_conn_log': +%% +%% ``` +%% {ct_conn_log,[{conn_mod(),hook_options()}]}. +%% ''' +%% +%% For example: +%% +%% ``` +%% {ct_conn_log,[{ct_telnet,[{log_type,raw}, +%% {hosts,[key_or_name()]}]}]} +%% ''' +%% +%% <b>Note</b> that hook options specified in a configuration file +%% will overwrite any hardcoded hook options in the test suite. +%% +%% === Logging example 1 === +%% +%% The following `ct_hooks' statement will cause raw printing of +%% telnet traffic to separate logs for the connections named +%% `server1' and `server2'. Any other connections will be logged +%% to default telnet log. +%% +%% ``` +%% suite() -> +%% [{ct_hooks, [{cth_conn_log, [{ct_telnet,[{log_type,raw}}, +%% {hosts,[server1,server2]}]} +%% ]}]}]. +%%''' +%% +%% === Logging example 2 === +%% +%% The following configuration file will cause raw logging of all +%% telnet traffic into one single text file. +%% +%% ``` +%% {ct_conn_log,[{ct_telnet,[{log_type,raw}]}]}. +%% ''' +%% +%% The `ct_hooks' statement must look like this: +%% +%% ``` +%% suite() -> +%% [{ct_hooks, [{cth_conn_log, []}]}]. +%% ''' +%% +%% The same `ct_hooks' statement without the configuration file would +%% cause HTML logging of all telnet connections into the test case +%% HTML log. +%% +%% <b>Note</b> that if the `cth_conn_log' hook is not added, telnet +%% traffic is still logged in the test case HTML log file (on the legacy +%% `ct_telnet' format). +%% @end --module(ct_telnet). +%% @type connection_type() = telnet | ts1 | ts2 + +%% @type connection() = handle() | +%% {ct:target_name(),connection_type()} | ct:target_name() --compile(export_all). +%% @type handle() = ct_gen_conn:handle(). Handle for a +%% specific telnet connection. + +%% @type prompt_regexp() = string(). A regular expression which +%% matches all possible prompts for a specific type of target. The +%% regexp must not have any groups i.e. when matching, re:run/3 shall +%% return a list with one single element. +%% +%% @see unix_telnet + +-module(ct_telnet). -export([open/1, open/2, open/3, open/4, close/1]). -export([cmd/2, cmd/3, cmdf/3, cmdf/4, get_data/1, @@ -71,10 +167,9 @@ -export([init/3,handle_msg/2,reconnect/2,terminate/2]). %% Tool internals --export([silent_teln_expect/5, teln_receive_until_prompt/3, - start_log/1, log/3, cont_log/2, end_log/0, - try_start_log/1, try_log/3, try_cont_log/2, try_end_log/0]). - +-export([silent_teln_expect/6, teln_receive_until_prompt/3, + format_data/2]). +-export([start_gen_log/1, end_gen_log/0, log/3, log/4]). -define(RECONNS,3). -define(RECONN_TIMEOUT,5000). @@ -83,12 +178,14 @@ -include("ct_util.hrl"). --record(state,{teln_pid, +-record(state,{host, + port, + teln_pid, prx, - type, buffer=[], prompt=false, name, + type, target_mod, keep_alive, extra, @@ -160,8 +257,7 @@ open(KeyOrName,ConnType,TargetMod) -> open(KeyOrName,ConnType,TargetMod,Extra) -> case ct:get_config({KeyOrName,ConnType}) of undefined -> - log(heading(open,{KeyOrName,ConnType}),"Failed: ~p", - [{not_available,KeyOrName}]), + log(undefined,open,"Failed: ~p",[{not_available,KeyOrName}]), {error,{not_available,KeyOrName,ConnType}}; Addr -> Addr1 = @@ -183,8 +279,8 @@ open(KeyOrName,ConnType,TargetMod,Extra) -> end; Bool -> Bool end, - log(heading(open,{KeyOrName,ConnType}), - "Opening connection to: ~p",[Addr1]), + log(undefined,open,"Connecting to ~p(~p)", + [KeyOrName,Addr1]), ct_gen_conn:start(KeyOrName,full_addr(Addr1,ConnType), {TargetMod,KeepAlive,Extra},?MODULE) end. @@ -202,7 +298,7 @@ open(KeyOrName,ConnType,TargetMod,Extra) -> close(Connection) -> case get_handle(Connection) of {ok,Pid} -> - log("ct_telnet:close","Handle: ~w",[Pid]), + log(undefined,close,"Connection closed, handle: ~w",[Pid]), case ct_gen_conn:stop(Pid) of {error,{process_down,Pid,noproc}} -> {error,already_closed}; @@ -408,9 +504,20 @@ init(Name,{Ip,Port,Type},{TargetMod,KeepAlive,Extra}) -> Settings -> set_telnet_defaults(Settings,#state{}) end, - case catch TargetMod:connect(Ip,Port,S0#state.conn_to,KeepAlive,Extra) of + case catch TargetMod:connect(Name,Ip,Port,S0#state.conn_to, + KeepAlive,Extra) of {ok,TelnPid} -> - log(heading(init,{Name,Type}), + put({ct_telnet_pid2name,TelnPid},Name), + S1 = S0#state{host=Ip, + port=Port, + teln_pid=TelnPid, + name=Name, + type=type(Type), + target_mod=TargetMod, + keep_alive=KeepAlive, + extra=Extra, + prx=TargetMod:get_prompt_regexp()}, + log(S1,open, "Opened telnet connection\n" "IP: ~p\n" "Port: ~p\n" @@ -419,15 +526,9 @@ init(Name,{Ip,Port,Type},{TargetMod,KeepAlive,Extra}) -> "Reconnection interval: ~p\n" "Connection timeout: ~p\n" "Keep alive: ~w", - [Ip,Port,S0#state.com_to,S0#state.reconns, - S0#state.reconn_int,S0#state.conn_to,KeepAlive]), - {ok,TelnPid,S0#state{teln_pid=TelnPid, - type=type(Type), - name={Name,Type}, - target_mod=TargetMod, - keep_alive=KeepAlive, - extra=Extra, - prx=TargetMod:get_prompt_regexp()}}; + [Ip,Port,S1#state.com_to,S1#state.reconns, + S1#state.reconn_int,S1#state.conn_to,KeepAlive]), + {ok,TelnPid,S1}; {'EXIT',Reason} -> {error,Reason}; Error -> @@ -448,27 +549,31 @@ set_telnet_defaults([{reconnection_interval,RInt}|Ss],S) -> set_telnet_defaults([{keep_alive,_}|Ss],S) -> set_telnet_defaults(Ss,S); set_telnet_defaults([Unknown|Ss],S) -> - log(heading(set_telnet_defaults,{telnet_settings,Unknown}), - "Bad element in telnet_settings: ~p",[Unknown]), + force_log(S,error, + "Bad element in telnet_settings: ~p",[Unknown]), set_telnet_defaults(Ss,S); set_telnet_defaults([],S) -> S. %% @hidden handle_msg({cmd,Cmd,Timeout},State) -> - try_start_log(heading(cmd,State#state.name)), - try_cont_log("Cmd: ~p", [Cmd]), - debug_cont_log("Throwing Buffer:",[]), + start_gen_log(heading(cmd,State#state.name)), + log(State,cmd,"Cmd: ~p",[Cmd]), + + debug_cont_gen_log("Throwing Buffer:",[]), debug_log_lines(State#state.buffer), + case {State#state.type,State#state.prompt} of {ts,_} -> - silent_teln_expect(State#state.teln_pid, + silent_teln_expect(State#state.name, + State#state.teln_pid, State#state.buffer, prompt, State#state.prx, [{timeout,2000}]); {ip,false} -> - silent_teln_expect(State#state.teln_pid, + silent_teln_expect(State#state.name, + State#state.teln_pid, State#state.buffer, prompt, State#state.prx, @@ -482,29 +587,36 @@ handle_msg({cmd,Cmd,Timeout},State) -> {Return,NewBuffer,Prompt} = case teln_cmd(State#state.teln_pid, Cmd, State#state.prx, TO) of {ok,Data,_PromptType,Rest} -> - try_cont_log("Return: ~p", [{ok,Data}]), + log(State,recv,"Return: ~p",[{ok,Data}]), {{ok,Data},Rest,true}; Error -> - Retry = {retry,{Error,State#state.name,State#state.teln_pid, + Retry = {retry,{Error, + {State#state.name, + State#state.type}, + State#state.teln_pid, {cmd,Cmd,TO}}}, - try_cont_log("Return: ~p", [Error]), + log(State,recv,"Return: ~p",[Error]), {Retry,[],false} end, - try_end_log(), + end_gen_log(), {Return,State#state{buffer=NewBuffer,prompt=Prompt}}; handle_msg({send,Cmd},State) -> - try_log(heading(send,State#state.name),"Cmd: ~p",[Cmd]), - debug_cont_log("Throwing Buffer:",[]), + log(State,send,"Cmd: ~p",[Cmd]), + + debug_cont_gen_log("Throwing Buffer:",[]), debug_log_lines(State#state.buffer), + case {State#state.type,State#state.prompt} of {ts,_} -> - silent_teln_expect(State#state.teln_pid, + silent_teln_expect(State#state.name, + State#state.teln_pid, State#state.buffer, prompt, State#state.prx, [{timeout,2000}]); {ip,false} -> - silent_teln_expect(State#state.teln_pid, + silent_teln_expect(State#state.name, + State#state.teln_pid, State#state.buffer, prompt, State#state.prx, @@ -515,19 +627,21 @@ handle_msg({send,Cmd},State) -> ct_telnet_client:send_data(State#state.teln_pid,Cmd), {ok,State#state{buffer=[],prompt=false}}; handle_msg(get_data,State) -> - try_start_log(heading(get_data,State#state.name)), + start_gen_log(heading(get_data,State#state.name)), + log(State,cmd,"Reading data...",[]), {ok,Data,Buffer} = teln_get_all_data(State#state.teln_pid, State#state.prx, State#state.buffer, [],[]), - try_cont_log("Return: ~p",[{ok,Data}]), - try_end_log(), + log(State,recv,"Return: ~p",[{ok,Data}]), + end_gen_log(), {{ok,Data},State#state{buffer=Buffer}}; handle_msg({expect,Pattern,Opts},State) -> - try_start_log(heading(expect,State#state.name)), - try_cont_log("Expect: ~p\nOpts=~p\n",[Pattern,Opts]), + start_gen_log(heading(expect,State#state.name)), + log(State,expect,"Expect: ~p\nOpts = ~p\n",[Pattern,Opts]), {Return,NewBuffer,Prompt} = - case teln_expect(State#state.teln_pid, + case teln_expect(State#state.name, + State#state.teln_pid, State#state.buffer, Pattern, State#state.prx, @@ -536,22 +650,23 @@ handle_msg({expect,Pattern,Opts},State) -> P = check_if_prompt_was_reached(Data,[]), {{ok,Data},Rest,P}; {ok,Data,HaltReason,Rest} -> - force_cont_log("HaltReason: ~p", - [HaltReason]), + force_log(State,expect,"HaltReason: ~p",[HaltReason]), P = check_if_prompt_was_reached(Data,HaltReason), {{ok,Data,HaltReason},Rest,P}; {error,Reason,Rest} -> - force_cont_log("Expect failed\n~p",[{error,Reason}]), + force_log(State,expect,"Expect failed\n~p",[{error,Reason}]), P = check_if_prompt_was_reached([],Reason), {{error,Reason},Rest,P}; {error,Reason} -> - force_cont_log("Expect failed\n~p",[{error,Reason}]), + force_log(State,expect,"Expect failed\n~p",[{error,Reason}]), P = check_if_prompt_was_reached([],Reason), {{error,Reason},[],P} end, - try_end_log(), + end_gen_log(), Return1 = case Return of - {error,_} -> {retry,{Return,State#state.name, + {error,_} -> {retry,{Return, + {State#state.name, + State#state.type}, State#state.teln_pid, {expect,Pattern,Opts}}}; _ -> Return @@ -562,18 +677,20 @@ handle_msg({expect,Pattern,Opts},State) -> %% @hidden reconnect({Ip,Port,_Type},State) -> reconnect(Ip,Port,State#state.reconns,State). -reconnect(Ip,Port,N,State=#state{target_mod=TargetMod, +reconnect(Ip,Port,N,State=#state{name=Name, + target_mod=TargetMod, keep_alive=KeepAlive, extra=Extra, conn_to=ConnTo, reconn_int=ReconnInt}) -> - case TargetMod:connect(Ip,Port,ConnTo,KeepAlive,Extra) of - {ok, NewPid} -> + case TargetMod:connect(Name,Ip,Port,ConnTo,KeepAlive,Extra) of + {ok,NewPid} -> + put({ct_telnet_pid2name,NewPid},Name), {ok, NewPid, State#state{teln_pid=NewPid}}; Error when N==0 -> Error; _Error -> - log("Reconnect failed!","Retries left: ~w",[N]), + log(State,reconnect,"Reconnect failed!","Retries left: ~w",[N]), timer:sleep(ReconnInt), reconnect(Ip,Port,N-1,State) end. @@ -581,11 +698,9 @@ reconnect(Ip,Port,N,State=#state{target_mod=TargetMod, %% @hidden terminate(TelnPid,State) -> - log(heading(terminate,State#state.name), - "Closing telnet connection.\nId: ~w", - [TelnPid]), - ct_telnet_client:close(TelnPid). - + Result = ct_telnet_client:close(TelnPid), + log(State,close,"Telnet connection for ~w closed.",[TelnPid]), + Result. %%%================================================================= %%% Internal function @@ -637,79 +752,109 @@ check_if_prompt_was_reached(Data,_) when is_list(Data) -> check_if_prompt_was_reached(_,_) -> false. -%tc(Fun) -> -% Before = erlang:now(), -% Val = Fun(), -% After = erlang:now(), -% {now_diff(After, Before), Val}. -%now_diff({A2, B2, C2}, {A1, B1, C1}) -> -% ((A2-A1)*1000000 + B2-B1)*1000000 + C2-C1. - -heading(Function,Name) -> - io_lib:format("~w:~w ~p",[?MODULE,Function,Name]). - -%%% @hidden -%% Functions for regular (unconditional) logging, to be -%% used during connect, reconnect, disconnect etc. -log(Heading,Str,Args) -> - ct_gen_conn:log(Heading,Str,Args). -%%% @hidden -start_log(Heading) -> - ct_gen_conn:start_log(Heading). -cont_log(Str,Args) -> - ct_gen_conn:cont_log(Str,Args). -end_log() -> - ct_gen_conn:end_log(). - -%%% @hidden -%% Functions for conditional logging, to be used by -%% cmd, send, receive, expect etc (this output may be -%% silenced by user). -try_start_log(Heading) -> - do_try_log(start_log,[Heading]). -%%% @hidden -try_end_log() -> - do_try_log(end_log,[]). - -%%% @hidden -try_log(Heading,Str,Args) -> - do_try_log(log,[Heading,Str,Args]). - -%%% @hidden -try_cont_log(Str,Args) -> - do_try_log(cont_log,[Str,Args]). - %%% @hidden -do_try_log(Func,Args) -> - %% check if output is suppressed - case ct_util:is_silenced(telnet) of - true -> +%% Functions for logging ct_telnet reports and telnet data + +heading(Action,undefined) -> + io_lib:format("~w ~w",[?MODULE,Action]); +heading(Action,Name) -> + io_lib:format("~w ~w for ~p",[?MODULE,Action,Name]). + +force_log(State,Action,String,Args) -> + log(State,Action,String,Args,true). + +log(State,Action,String,Args) when is_record(State, state) -> + log(State,Action,String,Args,false); +log(Name,Action,String,Args) when is_atom(Name) -> + log(#state{name=Name},Action,String,Args,false); +log(TelnPid,Action,String,Args) when is_pid(TelnPid) -> + log(#state{teln_pid=TelnPid},Action,String,Args,false). + +log(undefined,String,Args) -> + log(#state{},undefined,String,Args,false); +log(Name,String,Args) when is_atom(Name) -> + log(#state{name=Name},undefined,String,Args,false); +log(TelnPid,String,Args) when is_pid(TelnPid) -> + log(#state{teln_pid=TelnPid},undefined,String,Args). + +log(#state{name=Name,teln_pid=TelnPid,host=Host,port=Port}, + Action,String,Args,ForcePrint) -> + Name1 = if Name == undefined -> get({ct_telnet_pid2name,TelnPid}); + true -> Name + end, + Silent = get(silent), + case ct_util:get_testdata({cth_conn_log,?MODULE}) of + Result when Result /= undefined, Result /= silent, Silent /= true -> + {PrintHeader,PreBR} = if Action==undefined -> + {false,""}; + true -> + {true,"\n"} + end, + error_logger:info_report(#conn_log{header=PrintHeader, + client=self(), + conn_pid=TelnPid, + address={Host,Port}, + name=Name1, + action=Action, + module=?MODULE}, + {PreBR++String,Args}); + Result when Result /= undefined -> ok; - false -> - apply(ct_gen_conn,Func,Args) + _ when Action == open; Action == close; Action == reconnect; + Action == info; Action == error -> + ct_gen_conn:log(heading(Action,Name1),String,Args); + _ when ForcePrint == false -> + case ct_util:is_silenced(telnet) of + true -> + ok; + false -> + ct_gen_conn:cont_log(String,Args) + end; + _ when ForcePrint == true -> + case ct_util:is_silenced(telnet) of + true -> + %% call log/3 now instead of cont_log/2 since + %% start_gen_log/1 will not have been previously called + ct_gen_conn:log(heading(Action,Name1),String,Args); + false -> + ct_gen_conn:cont_log(String,Args) + end end. -%%% @hidden -%% Functions that will force printout even if ct_telnet -%% output has been silenced, to be used for error printouts. -force_cont_log(Str,Args) -> - case ct_util:is_silenced(telnet) of - true -> - %% call log/3 now instead of cont_log/2 since - %% start_log/1 will not have been previously called - log("ct_telnet info",Str,Args); - false -> - cont_log(Str,Args) +start_gen_log(Heading) -> + case ct_util:get_testdata({cth_conn_log,?MODULE}) of + undefined -> + %% check if output is suppressed + case ct_util:is_silenced(telnet) of + true -> ok; + false -> ct_gen_conn:start_log(Heading) + end; + _ -> + ok + end. + +end_gen_log() -> + case ct_util:get_testdata({cth_conn_log,?MODULE}) of + undefined -> + %% check if output is suppressed + case ct_util:is_silenced(telnet) of + true -> ok; + false -> ct_gen_conn:end_log() + end; + _ -> + ok end. %%% @hidden %% Debug printouts. -debug_cont_log(Str,Args) -> +debug_cont_gen_log(Str,Args) -> Old = put(silent,true), - cont_log(Str,Args), + ct_gen_conn:cont_log(Str,Args), put(silent,Old). - +%% Log callback - called from the error handler process +format_data(_How,{String,Args}) -> + io_lib:format(String,Args). %%%================================================================= %%% Abstraction layer on top of ct_telnet_client.erl @@ -717,7 +862,6 @@ teln_cmd(Pid,Cmd,Prx,Timeout) -> ct_telnet_client:send_data(Pid,Cmd), teln_receive_until_prompt(Pid,Prx,Timeout). - teln_get_all_data(Pid,Prx,Data,Acc,LastLine) -> case check_for_prompt(Prx,lists:reverse(LastLine) ++ Data) of {prompt,Lines,_PromptType,Rest} -> @@ -746,11 +890,9 @@ teln_get_all_data(Pid,Prx,Data,Acc,LastLine) -> %% @doc Externally the silent_teln_expect function shall only be used %% by the TargetModule, i.e. the target specific module which %% implements connect/2 and get_prompt_regexp/0. -silent_teln_expect(Pid,Data,Pattern,Prx,Opts) -> +silent_teln_expect(Name,Pid,Data,Pattern,Prx,Opts) -> Old = put(silent,true), - try_cont_log("silent_teln_expect/5, Pattern = ~p",[Pattern]), - Result = teln_expect(Pid,Data,Pattern,Prx,Opts), - try_cont_log("silent_teln_expect -> ~p\n",[Result]), + Result = teln_expect(Name,Pid,Data,Pattern,Prx,Opts), put(silent,Old), Result. @@ -766,7 +908,7 @@ silent_teln_expect(Pid,Data,Pattern,Prx,Opts) -> %% condition is fullfilled. %% 3b) Repeat (sequence): 2) is repeated either N times or until a %% halt condition is fullfilled. -teln_expect(Pid,Data,Pattern0,Prx,Opts) -> +teln_expect(Name,Pid,Data,Pattern0,Prx,Opts) -> HaltPatterns = case get_ignore_prompt(Opts) of true -> @@ -790,7 +932,7 @@ teln_expect(Pid,Data,Pattern0,Prx,Opts) -> case get_repeat(Opts) of false -> - case teln_expect1(Data,Pattern,[],EO) of + case teln_expect1(Name,Pid,Data,Pattern,[],EO) of {ok,Matched,Rest} -> {ok,Matched,Rest}; {halt,Why,Rest} -> @@ -800,7 +942,7 @@ teln_expect(Pid,Data,Pattern0,Prx,Opts) -> end; N -> EO1 = EO#eo{repeat=N}, - repeat_expect(Data,Pattern,[],EO1) + repeat_expect(Name,Pid,Data,Pattern,[],EO1) end. convert_pattern(Pattern,Seq) @@ -855,23 +997,27 @@ get_prompt_check(Opts) -> %% Repeat either single or sequence. All match results are accumulated %% and returned when a halt condition is fulllfilled. -repeat_expect(Rest,_Pattern,Acc,#eo{repeat=0}) -> +repeat_expect(_Name,_Pid,Rest,_Pattern,Acc,#eo{repeat=0}) -> {ok,lists:reverse(Acc),done,Rest}; -repeat_expect(Data,Pattern,Acc,EO) -> - case teln_expect1(Data,Pattern,[],EO) of +repeat_expect(Name,Pid,Data,Pattern,Acc,EO) -> + case teln_expect1(Name,Pid,Data,Pattern,[],EO) of {ok,Matched,Rest} -> EO1 = EO#eo{repeat=EO#eo.repeat-1}, - repeat_expect(Rest,Pattern,[Matched|Acc],EO1); + repeat_expect(Name,Pid,Rest,Pattern,[Matched|Acc],EO1); {halt,Why,Rest} -> {ok,lists:reverse(Acc),Why,Rest}; {error,Reason} -> {error,Reason} end. -teln_expect1(Data,Pattern,Acc,EO) -> +teln_expect1(Name,Pid,Data,Pattern,Acc,EO) -> ExpectFun = case EO#eo.seq of - true -> fun() -> seq_expect(Data,Pattern,Acc,EO) end; - false -> fun() -> one_expect(Data,Pattern,EO) end + true -> fun() -> + seq_expect(Name,Pid,Data,Pattern,Acc,EO) + end; + false -> fun() -> + one_expect(Name,Pid,Data,Pattern,EO) + end end, case ExpectFun() of {match,Match,Rest} -> @@ -890,10 +1036,10 @@ teln_expect1(Data,Pattern,Acc,EO) -> case NotFinished of {nomatch,Rest} -> %% One expect - teln_expect1(Rest++Data1,Pattern,[],EO); + teln_expect1(Name,Pid,Rest++Data1,Pattern,[],EO); {continue,Patterns1,Acc1,Rest} -> %% Sequence - teln_expect1(Rest++Data1,Patterns1,Acc1,EO) + teln_expect1(Name,Pid,Rest++Data1,Patterns1,Acc1,EO) end end end. @@ -913,47 +1059,45 @@ get_data1(Pid) -> %% lines and each line is matched against each pattern. %% one_expect: split data chunk at prompts -one_expect(Data,Pattern,EO) when EO#eo.prompt_check==false -> +one_expect(Name,Pid,Data,Pattern,EO) when EO#eo.prompt_check==false -> % io:format("Raw Data ~p Pattern ~p EO ~p ",[Data,Pattern,EO]), - one_expect1(Data,Pattern,[],EO#eo{found_prompt=false}); -one_expect(Data,Pattern,EO) -> + one_expect1(Name,Pid,Data,Pattern,[],EO#eo{found_prompt=false}); +one_expect(Name,Pid,Data,Pattern,EO) -> case match_prompt(Data,EO#eo.prx) of {prompt,UptoPrompt,PromptType,Rest} -> case Pattern of [Prompt] when Prompt==prompt; Prompt=={prompt,PromptType} -> %% Only searching for prompt - log_lines(UptoPrompt), - try_cont_log("<b>PROMPT:</b> ~ts", [PromptType]), + log_lines(Name,Pid,UptoPrompt), + log(name_or_pid(Name,Pid),"PROMPT: ~ts",[PromptType]), {match,{prompt,PromptType},Rest}; [{prompt,_OtherPromptType}] -> %% Only searching for one specific prompt, not thisone - log_lines(UptoPrompt), + log_lines(Name,Pid,UptoPrompt), {nomatch,Rest}; _ -> - one_expect1(UptoPrompt,Pattern,Rest, + one_expect1(Name,Pid,UptoPrompt,Pattern,Rest, EO#eo{found_prompt=PromptType}) end; noprompt -> case Pattern of [Prompt] when Prompt==prompt; element(1,Prompt)==prompt -> %% Only searching for prompt - LastLine = log_lines_not_last(Data), + LastLine = log_lines_not_last(Name,Pid,Data), {nomatch,LastLine}; _ -> - one_expect1(Data,Pattern,[],EO#eo{found_prompt=false}) + one_expect1(Name,Pid,Data,Pattern,[], + EO#eo{found_prompt=false}) end end. -remove_zero(List) -> - [Ch || Ch <- List, Ch=/=0, Ch=/=13]. - %% one_expect1: split data chunk at lines -one_expect1(Data,Pattern,Rest,EO) -> - case match_lines(Data,Pattern,EO) of +one_expect1(Name,Pid,Data,Pattern,Rest,EO) -> + case match_lines(Name,Pid,Data,Pattern,EO) of {match,Match,MatchRest} -> {match,Match,MatchRest++Rest}; {nomatch,prompt} -> - one_expect(Rest,Pattern,EO); + one_expect(Name,Pid,Rest,Pattern,EO); {nomatch,NoMatchRest} -> {nomatch,NoMatchRest++Rest}; {halt,Why,HaltRest} -> @@ -970,77 +1114,77 @@ one_expect1(Data,Pattern,Rest,EO) -> %% searching for the next pattern in the list. %% seq_expect: Split data chunk at prompts -seq_expect(Data,[],Acc,_EO) -> +seq_expect(_Name,_Pid,Data,[],Acc,_EO) -> {match,lists:reverse(Acc),Data}; -seq_expect([],Patterns,Acc,_EO) -> +seq_expect(_Name,_Pid,[],Patterns,Acc,_EO) -> {continue,Patterns,lists:reverse(Acc),[]}; -seq_expect(Data,Patterns,Acc,EO) when EO#eo.prompt_check==false -> - seq_expect1(Data,Patterns,Acc,[],EO#eo{found_prompt=false}); -seq_expect(Data,Patterns,Acc,EO) -> +seq_expect(Name,Pid,Data,Patterns,Acc,EO) when EO#eo.prompt_check==false -> + seq_expect1(Name,Pid,Data,Patterns,Acc,[],EO#eo{found_prompt=false}); +seq_expect(Name,Pid,Data,Patterns,Acc,EO) -> case match_prompt(Data,EO#eo.prx) of {prompt,UptoPrompt,PromptType,Rest} -> - seq_expect1(UptoPrompt,Patterns,Acc,Rest, + seq_expect1(Name,Pid,UptoPrompt,Patterns,Acc,Rest, EO#eo{found_prompt=PromptType}); noprompt -> - seq_expect1(Data,Patterns,Acc,[],EO#eo{found_prompt=false}) + seq_expect1(Name,Pid,Data,Patterns,Acc,[],EO#eo{found_prompt=false}) end. %% seq_expect1: For one prompt-chunk, match each pattern - line by %% line if it is other than the prompt we are seaching for. -seq_expect1(Data,[prompt|Patterns],Acc,Rest,EO) -> +seq_expect1(Name,Pid,Data,[prompt|Patterns],Acc,Rest,EO) -> case EO#eo.found_prompt of false -> - LastLine = log_lines_not_last(Data), + LastLine = log_lines_not_last(Name,Pid,Data), %% Rest==[] because no prompt is found {continue,[prompt|Patterns],Acc,LastLine}; PromptType -> - log_lines(Data), - try_cont_log("<b>PROMPT:</b> ~ts", [PromptType]), - seq_expect(Rest,Patterns,[{prompt,PromptType}|Acc],EO) + log_lines(Name,Pid,Data), + log(name_or_pid(Name,Pid),"PROMPT: ~ts",[PromptType]), + seq_expect(Name,Pid,Rest,Patterns,[{prompt,PromptType}|Acc],EO) end; -seq_expect1(Data,[{prompt,PromptType}|Patterns],Acc,Rest,EO) -> +seq_expect1(Name,Pid,Data,[{prompt,PromptType}|Patterns],Acc,Rest,EO) -> case EO#eo.found_prompt of false -> - LastLine = log_lines_not_last(Data), + LastLine = log_lines_not_last(Name,Pid,Data), %% Rest==[] because no prompt is found {continue,[{prompt,PromptType}|Patterns],Acc,LastLine}; PromptType -> - log_lines(Data), - try_cont_log("<b>PROMPT:</b> ~ts", [PromptType]), - seq_expect(Rest,Patterns,[{prompt,PromptType}|Acc],EO); + log_lines(Name,Pid,Data), + log(name_or_pid(Name,Pid),"PROMPT: ~ts", [PromptType]), + seq_expect(Name,Pid,Rest,Patterns,[{prompt,PromptType}|Acc],EO); _OtherPromptType -> - log_lines(Data), - seq_expect(Rest,[{prompt,PromptType}|Patterns],Acc,EO) + log_lines(Name,Pid,Data), + seq_expect(Name,Pid,Rest,[{prompt,PromptType}|Patterns],Acc,EO) end; -seq_expect1(Data,[Pattern|Patterns],Acc,Rest,EO) -> - case match_lines(Data,[Pattern],EO) of +seq_expect1(Name,Pid,Data,[Pattern|Patterns],Acc,Rest,EO) -> + case match_lines(Name,Pid,Data,[Pattern],EO) of {match,Match,MatchRest} -> - seq_expect1(MatchRest,Patterns,[Match|Acc],Rest,EO); + seq_expect1(Name,Pid,MatchRest,Patterns,[Match|Acc],Rest,EO); {nomatch,prompt} -> - seq_expect(Rest,[Pattern|Patterns],Acc,EO); + seq_expect(Name,Pid,Rest,[Pattern|Patterns],Acc,EO); {nomatch,NoMatchRest} when Rest==[] -> %% The data did not end with a prompt {continue,[Pattern|Patterns],Acc,NoMatchRest}; {halt,Why,HaltRest} -> {halt,Why,HaltRest++Rest} end; -seq_expect1(Data,[],Acc,Rest,_EO) -> +seq_expect1(_Name,_Pid,Data,[],Acc,Rest,_EO) -> {match,lists:reverse(Acc),Data++Rest}. %% Split prompt-chunk at lines -match_lines(Data,Patterns,EO) -> +match_lines(Name,Pid,Data,Patterns,EO) -> FoundPrompt = EO#eo.found_prompt, case one_line(Data,[]) of {noline,Rest} when FoundPrompt=/=false -> %% This is the line including the prompt - case match_line(Rest,Patterns,FoundPrompt,EO) of + case match_line(Name,Pid,Rest,Patterns,FoundPrompt,EO) of nomatch -> {nomatch,prompt}; {Tag,Match} -> {Tag,Match,[]} end; {noline,Rest} when EO#eo.prompt_check==false -> - case match_line(Rest,Patterns,false,EO) of + case match_line(Name,Pid,Rest,Patterns,false,EO) of nomatch -> {nomatch,Rest}; {Tag,Match} -> @@ -1049,9 +1193,9 @@ match_lines(Data,Patterns,EO) -> {noline,Rest} -> {nomatch,Rest}; {Line,Rest} -> - case match_line(Line,Patterns,false,EO) of + case match_line(Name,Pid,Line,Patterns,false,EO) of nomatch -> - match_lines(Rest,Patterns,EO); + match_lines(Name,Pid,Rest,Patterns,EO); {Tag,Match} -> {Tag,Match,Rest} end @@ -1059,43 +1203,43 @@ match_lines(Data,Patterns,EO) -> %% For one line, match each pattern -match_line(Line,Patterns,FoundPrompt,EO) -> - match_line(Line,Patterns,FoundPrompt,EO,match). - -match_line(Line,[prompt|Patterns],false,EO,RetTag) -> - match_line(Line,Patterns,false,EO,RetTag); -match_line(Line,[prompt|_Patterns],FoundPrompt,_EO,RetTag) -> - try_cont_log(" ~ts", [Line]), - try_cont_log("<b>PROMPT:</b> ~ts", [FoundPrompt]), +match_line(Name,Pid,Line,Patterns,FoundPrompt,EO) -> + match_line(Name,Pid,Line,Patterns,FoundPrompt,EO,match). + +match_line(Name,Pid,Line,[prompt|Patterns],false,EO,RetTag) -> + match_line(Name,Pid,Line,Patterns,false,EO,RetTag); +match_line(Name,Pid,Line,[prompt|_Patterns],FoundPrompt,_EO,RetTag) -> + log(name_or_pid(Name,Pid)," ~ts",[Line]), + log(name_or_pid(Name,Pid),"PROMPT: ~ts",[FoundPrompt]), {RetTag,{prompt,FoundPrompt}}; -match_line(Line,[{prompt,PromptType}|_Patterns],FoundPrompt,_EO,RetTag) +match_line(Name,Pid,Line,[{prompt,PromptType}|_Patterns],FoundPrompt,_EO,RetTag) when PromptType==FoundPrompt -> - try_cont_log(" ~ts", [Line]), - try_cont_log("<b>PROMPT:</b> ~ts", [FoundPrompt]), + log(name_or_pid(Name,Pid)," ~ts",[Line]), + log(name_or_pid(Name,Pid),"PROMPT: ~ts",[FoundPrompt]), {RetTag,{prompt,FoundPrompt}}; -match_line(Line,[{prompt,PromptType}|Patterns],FoundPrompt,EO,RetTag) +match_line(Name,Pid,Line,[{prompt,PromptType}|Patterns],FoundPrompt,EO,RetTag) when PromptType=/=FoundPrompt -> - match_line(Line,Patterns,FoundPrompt,EO,RetTag); -match_line(Line,[{Tag,Pattern}|Patterns],FoundPrompt,EO,RetTag) -> + match_line(Name,Pid,Line,Patterns,FoundPrompt,EO,RetTag); +match_line(Name,Pid,Line,[{Tag,Pattern}|Patterns],FoundPrompt,EO,RetTag) -> case re:run(Line,Pattern,[{capture,all,list}]) of nomatch -> - match_line(Line,Patterns,FoundPrompt,EO,RetTag); + match_line(Name,Pid,Line,Patterns,FoundPrompt,EO,RetTag); {match,Match} -> - try_cont_log("<b>MATCH:</b> ~ts", [Line]), + log(name_or_pid(Name,Pid),"MATCH: ~ts",[Line]), {RetTag,{Tag,Match}} end; -match_line(Line,[Pattern|Patterns],FoundPrompt,EO,RetTag) -> +match_line(Name,Pid,Line,[Pattern|Patterns],FoundPrompt,EO,RetTag) -> case re:run(Line,Pattern,[{capture,all,list}]) of nomatch -> - match_line(Line,Patterns,FoundPrompt,EO,RetTag); + match_line(Name,Pid,Line,Patterns,FoundPrompt,EO,RetTag); {match,Match} -> - try_cont_log("<b>MATCH:</b> ~ts", [Line]), + log(name_or_pid(Name,Pid),"MATCH: ~ts",[Line]), {RetTag,Match} end; -match_line(Line,[],FoundPrompt,EO,match) -> - match_line(Line,EO#eo.haltpatterns,FoundPrompt,EO,halt); -match_line(Line,[],_FoundPrompt,_EO,halt) -> - try_cont_log(" ~ts", [Line]), +match_line(Name,Pid,Line,[],FoundPrompt,EO,match) -> + match_line(Name,Pid,Line,EO#eo.haltpatterns,FoundPrompt,EO,halt); +match_line(Name,Pid,Line,[],_FoundPrompt,_EO,halt) -> + log(name_or_pid(Name,Pid)," ~ts",[Line]), nomatch. one_line([$\n|Rest],Line) -> @@ -1111,26 +1255,29 @@ one_line([],Line) -> debug_log_lines(String) -> Old = put(silent,true), - log_lines(String), + log_lines(undefined,undefined,String), put(silent,Old). -log_lines(String) -> - case log_lines_not_last(String) of +log_lines(Name,Pid,String) -> + case log_lines_not_last(Name,Pid,String) of [] -> ok; LastLine -> - try_cont_log(" ~ts", [LastLine]) + log(name_or_pid(Name,Pid)," ~ts",[LastLine]) end. -log_lines_not_last(String) -> +log_lines_not_last(Name,Pid,String) -> case add_tabs(String,[],[]) of {[],LastLine} -> LastLine; {String1,LastLine} -> - try_cont_log("~ts",[String1]), + log(name_or_pid(Name,Pid),"~ts",[String1]), LastLine end. +name_or_pid(undefined,Pid) -> Pid; +name_or_pid(Name,_) -> Name. + add_tabs([0|Rest],Acc,LastLine) -> add_tabs(Rest,Acc,LastLine); add_tabs([$\r|Rest],Acc,LastLine) -> @@ -1145,8 +1292,6 @@ add_tabs([],[],LastLine) -> {[],lists:reverse(LastLine)}. - - %%% @hidden teln_receive_until_prompt(Pid,Prx,Timeout) -> Fun = fun() -> teln_receive_until_prompt(Pid,Prx,[],[]) end, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index bcc4caa62e..ac9857487d 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -382,9 +382,18 @@ loop(Mode,TestData,StartDir) -> TestData1 = case lists:keysearch(Key,1,TestData) of {value,{Key,Val}} -> - NewVal = Fun(Val), - return(From,NewVal), - [{Key,NewVal}|lists:keydelete(Key,1,TestData)]; + try Fun(Val) of + '$delete' -> + return(From,deleted), + lists:keydelete(Key,1,TestData); + NewVal -> + return(From,NewVal), + [{Key,NewVal}|lists:keydelete(Key,1,TestData)] + catch + _:Error -> + return(From,{error,Error}), + TestData + end; _ -> case lists:member(create,Opts) of true -> diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 7c01e17c36..a82d58cc42 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -79,4 +79,10 @@ -define(tablesorter_script, "jquery.tablesorter.min.js"). %% Logging information for error handler --record(conn_log, {client, name, address, action, module}). +-record(conn_log, {header=true, + client, + name, + address, + conn_pid, + action, + module}). diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index 644594e34d..a731c8054c 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -56,11 +56,29 @@ pre_init_per_testcase/3, post_end_per_testcase/4]). +%%---------------------------------------------------------------------- +%% Exported types +%%---------------------------------------------------------------------- +-export_type([hook_options/0, + log_type/0, + conn_mod/0]). + +%%---------------------------------------------------------------------- +%% Type declarations +%%---------------------------------------------------------------------- +-type hook_options() :: [hook_option()]. +%% Options that can be given to `cth_conn_log' in the `ct_hook' statement. +-type hook_option() :: {log_type,log_type()} | + {hosts,[ct_gen_conn:key_or_name()]}. +-type log_type() :: raw | pretty | html | silent. +-type conn_mod() :: ct_netconfc | ct_telnet. +%%---------------------------------------------------------------------- + -spec init(Id, HookOpts) -> Result when Id :: term(), - HookOpts :: ct_netconfc:hook_options(), - Result :: {ok,[{ct_netconfc:conn_mod(), - {ct_netconfc:log_type(),[ct_netconfc:key_or_name()]}}]}. + HookOpts :: hook_options(), + Result :: {ok,[{conn_mod(), + {log_type(),[ct_gen_conn:key_or_name()]}}]}. init(_Id, HookOpts) -> ConfOpts = ct:get_config(ct_conn_log,[]), {ok,merge_log_info(ConfOpts,HookOpts)}. @@ -86,7 +104,8 @@ get_log_opts(Opts) -> pre_init_per_testcase(TestCase,Config,CthState) -> Logs = lists:map( - fun({ConnMod,{LogType,Hosts}}) -> + fun({ConnMod,{LogType,Hosts}}) -> + ct_util:set_testdata({{?MODULE,ConnMod},LogType}), case LogType of LogType when LogType==raw; LogType==pretty -> Dir = ?config(priv_dir,Config), @@ -117,9 +136,44 @@ pre_init_per_testcase(TestCase,Config,CthState) -> end end, CthState), - error_logger:add_report_handler(ct_conn_log_h,{group_leader(),Logs}), + + GL = group_leader(), + Update = + fun(Init) when Init == undefined; Init == [] -> + error_logger:add_report_handler(ct_conn_log_h,{GL,Logs}), + [TestCase]; + (PrevUsers) -> + error_logger:info_report(update,{GL,Logs}), + receive + {updated,GL} -> + [TestCase|PrevUsers] + after + 5000 -> + {error,no_response} + end + end, + ct_util:update_testdata(?MODULE, Update, [create]), {Config,CthState}. -post_end_per_testcase(_TestCase,_Config,Return,CthState) -> - error_logger:delete_report_handler(ct_conn_log_h), +post_end_per_testcase(TestCase,_Config,Return,CthState) -> + Update = + fun(PrevUsers) -> + case lists:delete(TestCase, PrevUsers) of + [] -> + '$delete'; + PrevUsers1 -> + PrevUsers1 + end + end, + case ct_util:update_testdata(?MODULE, Update) of + deleted -> + [ct_util:delete_testdata({?MODULE,ConnMod}) || + {ConnMod,_} <- CthState], + error_logger:delete_report_handler(ct_conn_log_h); + {error,no_response} -> + exit({?MODULE,no_response_from_logger}); + _PrevUsers -> + ok + end, {Return,CthState}. + diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 8fed341600..61700a2032 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -73,7 +73,7 @@ pre_init_per_group(Group, Config, State) -> set_curr_func({group,Group,init_per_group}, Config), {Config, State}. -post_init_per_group(Group, Config, Result, tc_log_async) -> +post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) -> case lists:member(parallel,proplists:get_value( tc_group_properties,Config,[])) of true -> @@ -154,7 +154,8 @@ handle_info(_, State) -> handle_call(flush,State) -> {ok, ok, State}; -handle_call({set_curr_func,{group,Group,Conf},Config}, State) -> +handle_call({set_curr_func,{group,Group,Conf},Config}, + State) when is_list(Config) -> Parallel = case proplists:get_value(tc_group_properties, Config) of undefined -> false; Props -> lists:member(parallel, Props) @@ -162,6 +163,10 @@ handle_call({set_curr_func,{group,Group,Conf},Config}, State) -> {ok, ok, State#eh_state{curr_group = Group, curr_func = Conf, parallel_tcs = Parallel}}; +handle_call({set_curr_func,{group,Group,Conf},_SkipOrFail}, State) -> + {ok, ok, State#eh_state{curr_group = Group, + curr_func = Conf, + parallel_tcs = false}}; handle_call({set_curr_func,{group,undefined},_Config}, State) -> {ok, ok, State#eh_state{curr_group = undefined, curr_func = undefined, diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 1a38b6584b..7ed2018bdf 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -138,6 +138,9 @@ on_tc_fail(_TC, Res, State) -> {fail,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. +on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0) + when Type == tc_auto_skip; Type == tc_user_skip -> + on_tc_skip(ConfigFunc, Res, State0); on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> TcStr = atom_to_list(Tc), State = @@ -330,5 +333,7 @@ count_tcs([#testcase{result={fail,_}}|TCs],Ok,F,S) -> count_tcs(TCs,Ok,F+1,S); count_tcs([#testcase{result={skipped,_}}|TCs],Ok,F,S) -> count_tcs(TCs,Ok,F,S+1); +count_tcs([#testcase{result={auto_skipped,_}}|TCs],Ok,F,S) -> + count_tcs(TCs,Ok,F,S+1); count_tcs([],Ok,F,S) -> {Ok+F+S,F,S}. diff --git a/lib/common_test/src/unix_telnet.erl b/lib/common_test/src/unix_telnet.erl index 88199b07d0..e049c3bf39 100644 --- a/lib/common_test/src/unix_telnet.erl +++ b/lib/common_test/src/unix_telnet.erl @@ -54,8 +54,8 @@ -compile(export_all). %% Callbacks for ct_telnet.erl --export([connect/5,get_prompt_regexp/0]). --import(ct_telnet,[start_log/1,cont_log/2,end_log/0]). +-export([connect/6,get_prompt_regexp/0]). +-import(ct_telnet,[start_gen_log/1,log/4,end_gen_log/0]). -define(username,"login: "). -define(password,"Password: "). @@ -76,7 +76,9 @@ get_prompt_regexp() -> %%%----------------------------------------------------------------- %%% @hidden -%%% @spec connect(Ip,Port,Timeout,KeepAlive,Extra) -> {ok,Handle} | {error,Reason} +%%% @spec connect(ConnName,Ip,Port,Timeout,KeepAlive,Extra) -> +%%% {ok,Handle} | {error,Reason} +%%% ConnName = ct:target_name() %%% Ip = string() | {integer(),integer(),integer(),integer()} %%% Port = integer() %%% Timeout = integer() @@ -89,59 +91,70 @@ get_prompt_regexp() -> %%% @doc Callback for ct_telnet.erl. %%% %%% <p>Setup telnet connection to a UNIX host.</p> -connect(Ip,Port,Timeout,KeepAlive,Extra) -> +connect(ConnName,Ip,Port,Timeout,KeepAlive,Extra) -> case Extra of {Username,Password} -> - connect1(Ip,Port,Timeout,KeepAlive,Username,Password); - Name -> - case get_username_and_password(Name) of + connect1(ConnName,Ip,Port,Timeout,KeepAlive, + Username,Password); + KeyOrName -> + case get_username_and_password(KeyOrName) of {ok,{Username,Password}} -> - connect1(Ip,Port,Timeout,KeepAlive,Username,Password); + connect1(ConnName,Ip,Port,Timeout,KeepAlive, + Username,Password); Error -> Error end end. -connect1(Ip,Port,Timeout,KeepAlive,Username,Password) -> - start_log("unix_telnet:connect"), +connect1(Name,Ip,Port,Timeout,KeepAlive,Username,Password) -> + start_gen_log("unix_telnet connect"), Result = case ct_telnet_client:open(Ip,Port,Timeout,KeepAlive) of {ok,Pid} -> - case ct_telnet:silent_teln_expect(Pid,[],[prompt],?prx,[]) of + case ct_telnet:silent_teln_expect(Name,Pid,[], + [prompt],?prx,[]) of {ok,{prompt,?username},_} -> + log(Name,send,"Logging in to ~p:~p", [Ip,Port]), ok = ct_telnet_client:send_data(Pid,Username), - cont_log("Username: ~ts",[Username]), - case ct_telnet:silent_teln_expect(Pid,[],prompt,?prx,[]) of + log(Name,send,"Username: ~ts",[Username]), + case ct_telnet:silent_teln_expect(Name,Pid,[], + prompt,?prx,[]) of {ok,{prompt,?password},_} -> ok = ct_telnet_client:send_data(Pid,Password), Stars = lists:duplicate(length(Password),$*), - cont_log("Password: ~s",[Stars]), + log(Name,send,"Password: ~s",[Stars]), ok = ct_telnet_client:send_data(Pid,""), - case ct_telnet:silent_teln_expect(Pid,[],prompt, + case ct_telnet:silent_teln_expect(Name,Pid,[], + prompt, ?prx,[]) of {ok,{prompt,Prompt},_} - when Prompt=/=?username, Prompt=/=?password -> + when Prompt=/=?username, + Prompt=/=?password -> {ok,Pid}; Error -> - cont_log("Password failed\n~p\n", - [Error]), + log(Name,recv,"Password failed\n~p\n", + [Error]), {error,Error} end; Error -> - cont_log("Login failed\n~p\n",[Error]), + log(Name,recv,"Login to ~p:~p failed\n~p\n",[Ip,Port,Error]), {error,Error} end; {ok,[{prompt,_OtherPrompt1},{prompt,_OtherPrompt2}],_} -> {ok,Pid}; Error -> - cont_log("Did not get expected prompt\n~p\n",[Error]), + log(Name,error, + "Did not get expected prompt from ~p:~p\n~p\n", + [Ip,Port,Error]), {error,Error} end; Error -> - cont_log("Could not open telnet connection\n~p\n",[Error]), + log(Name,error, + "Could not open telnet connection to ~p:~p\n~p\n", + [Ip,Port,Error]), Error end, - end_log(), + end_gen_log(), Result. get_username_and_password(Name) -> diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index d92be9ec6e..284f836517 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -234,25 +234,25 @@ expected_events(config_static_SUITE)-> ?sok(test_get_config_deep_nested,{3,0,{0,0}}), ?sok(test_default_suitewide,{4,0,{0,0}}), ?snok(test_config_name_already_in_use1, - {skipped,{config_name_already_in_use,[x1]}},{4,0,{1,0}}), - ?sok(test_default_tclocal,{5,0,{1,0}}), + {failed,{error,{config_name_already_in_use,[x1]}}},{4,1,{0,0}}), + ?sok(test_default_tclocal,{5,1,{0,0}}), ?snok(test_config_name_already_in_use2, - {skipped,{config_name_already_in_use,[alias,x1]}},{5,0,{2,0}}), - ?sok(test_alias_tclocal,{6,0,{2,0}}), - ?sok(test_get_config_undefined,{7,0,{2,0}}), - ?sok(test_require_subvals,{8,0,{2,0}}), + {failed,{error,{config_name_already_in_use,[alias,x1]}}},{5,2,{0,0}}), + ?sok(test_alias_tclocal,{6,2,{0,0}}), + ?sok(test_get_config_undefined,{7,2,{0,0}}), + ?sok(test_require_subvals,{8,2,{0,0}}), ?snok(test_require_subvals2, - {skipped,{require_failed, - {not_available,{gen_cfg,[a,b,c,d]}}}},{8,0,{2,1}}), - ?sok(test_require_deep_config,{9,0,{2,1}}), - ?sok(test_shadow_all,{10,0,{2,1}}), - ?sok(test_element,{11,0,{2,1}}), - ?sok(test_shadow_all_element,{12,0,{2,1}}), - ?sok(test_internal_deep,{13,0,{2,1}}), - ?sok(test_alias_tclocal_nested,{14,0,{2,1}}), - ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}), - ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}), - ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}), + {auto_skipped,{require_failed, + {not_available,{gen_cfg,[a,b,c,d]}}}},{8,2,{0,1}}), + ?sok(test_require_deep_config,{9,2,{0,1}}), + ?sok(test_shadow_all,{10,2,{0,1}}), + ?sok(test_element,{11,2,{0,1}}), + ?sok(test_shadow_all_element,{12,2,{0,1}}), + ?sok(test_internal_deep,{13,2,{0,1}}), + ?sok(test_alias_tclocal_nested,{14,2,{0,1}}), + ?sok(test_alias_tclocal_nested_backward_compat,{15,2,{0,1}}), + ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,2,{0,1}}), + ?sok(test_config_same_name_already_in_use,{17,2,{0,1}}), {?eh,tc_start,{config_static_SUITE,end_per_suite}}, {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl index 10fe8286dd..8f2f0eb75f 100644 --- a/lib/common_test/test/ct_config_info_SUITE.erl +++ b/lib/common_test/test/ct_config_info_SUITE.erl @@ -127,7 +127,7 @@ test_events(config_info) -> {failed,{timetrap_timeout,350}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t11, {failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}}, - {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g1}, {failed,{config_info_1_SUITE,init_per_group, {timetrap_timeout,350}}}}}], @@ -145,12 +145,12 @@ test_events(config_info) -> {?eh,tc_auto_skip,{config_info_1_SUITE,t41, {failed,{config_info_1_SUITE,init_per_group, {timetrap_timeout,400}}}}}, - {?eh,tc_auto_skip,{config_info_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{config_info_1_SUITE,{end_per_group,g4}, {failed,{config_info_1_SUITE,init_per_group, {timetrap_timeout,400}}}}}], {?eh,tc_start,{config_info_1_SUITE,t31}}, {?eh,tc_done,{config_info_1_SUITE,t31, - {skipped,{failed,{config_info_1_SUITE,init_per_testcase, + {auto_skipped,{failed,{config_info_1_SUITE,init_per_testcase, {timetrap_timeout,250}}}}}}, {?eh,tc_start,{config_info_1_SUITE,t32}}, {?eh,tc_done,{config_info_1_SUITE,t32, diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 3881ced17d..194e7d42ae 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -66,7 +66,7 @@ all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, timetrap_normal, timetrap_extended, timetrap_parallel, timetrap_fun, timetrap_fun_group, misc_errors, - config_restored]. + config_restored, config_func_errors]. groups() -> []. @@ -310,6 +310,25 @@ config_restored(Config) when is_list(Config) -> ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- +%%% +config_func_errors(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "error/test/config_func_error_1_SUITE"), + {Opts,ERPid} = setup([{suite,Suite}], + Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(config_func_errors, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(config_func_errors), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -323,8 +342,6 @@ setup(Test, Config) -> reformat(Events, EH) -> ct_test_support:reformat(Events, EH). - %reformat(Events, _EH) -> - % Events. %%%----------------------------------------------------------------- %%% TEST EVENTS @@ -465,7 +482,7 @@ test_events(cfg_error) -> {'EXIT',{init_per_group_fails,g1}}}}}}, {?eh,test_stats,{4,0,{0,11}}}, {?eh,tc_auto_skip, - {cfg_error_8_SUITE,end_per_group, + {cfg_error_8_SUITE,{end_per_group,g1}, {failed,{cfg_error_8_SUITE,init_per_group, {'EXIT',{init_per_group_fails,g1}}}}}}], @@ -476,7 +493,7 @@ test_events(cfg_error) -> {failed,{cfg_error_8_SUITE,init_per_group, {timetrap_timeout,2000}}}}}, {?eh,test_stats,{4,0,{0,12}}}, - {?eh,tc_auto_skip,{cfg_error_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{cfg_error_8_SUITE,{end_per_group,g2}, {failed,{cfg_error_8_SUITE,init_per_group, {timetrap_timeout,2000}}}}}], @@ -490,7 +507,7 @@ test_events(cfg_error) -> {'EXIT',{{badmatch,42},'_'}}}}}}, {?eh,test_stats,{4,0,{0,13}}}, {?eh,tc_auto_skip, - {cfg_error_8_SUITE,end_per_group, + {cfg_error_8_SUITE,{end_per_group,g3}, {failed,{cfg_error_8_SUITE,init_per_group, {'EXIT',{{badmatch,42},'_'}}}}}}], @@ -516,7 +533,7 @@ test_events(cfg_error) -> {'EXIT',{sub_group_failed,g6}}}}}}, {?eh,test_stats,{6,0,{0,14}}}, {?eh,tc_auto_skip, - {cfg_error_8_SUITE,end_per_group, + {cfg_error_8_SUITE,{end_per_group,g6}, {failed,{cfg_error_8_SUITE,init_per_group, {'EXIT',{sub_group_failed,g6}}}}}}], {?eh,tc_start,{cfg_error_8_SUITE,tc3}}, @@ -550,23 +567,23 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_9_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_9_SUITE,tc1}}, {?eh,tc_done,{cfg_error_9_SUITE,tc1, - {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, - {tc1_should_be_skipped,'_'}}}}}}, + {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, + {tc1_should_be_skipped,'_'}}}}}}, {?eh,test_stats,{9,0,{0,15}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc2}}, {?eh,tc_done,{cfg_error_9_SUITE,tc2, - {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, - {timetrap_timeout,2000}}}}}}, + {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, + {timetrap_timeout,2000}}}}}}, {?eh,test_stats,{9,0,{0,16}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc3}}, {?eh,tc_done,{cfg_error_9_SUITE,tc3, - {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, - {{badmatch,undefined},'_'}}}}}}, + {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase, + {{badmatch,undefined},'_'}}}}}}, {?eh,test_stats,{9,0,{0,17}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc4}}, {?eh,tc_done, {cfg_error_9_SUITE,tc4, - {skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,bad_return}}}}}, + {auto_skipped,{failed,{cfg_error_9_SUITE,init_per_testcase,bad_return}}}}}, {?eh,test_stats,{9,0,{0,18}}}, {?eh,tc_start,{cfg_error_9_SUITE,tc5}}, {?eh,tc_done, @@ -622,41 +639,41 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_11_SUITE,tc1}}, {?eh,tc_done, {cfg_error_11_SUITE,tc1, - {skipped,{config_name_already_in_use,[dummy_alias]}}}}, - {?eh,test_stats,{12,6,{1,19}}}, + {failed,{error,{config_name_already_in_use,[dummy_alias]}}}}}, + {?eh,test_stats,{12,7,{0,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,tc2}}, {?eh,tc_done,{cfg_error_11_SUITE,tc2,ok}}, - {?eh,test_stats,{13,6,{1,19}}}, + {?eh,test_stats,{13,7,{0,19}}}, {?eh,tc_start,{cfg_error_11_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_11_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_12_SUITE,tc1}}, {?eh,tc_done,{ct_framework,init_tc,{framework_error,{timetrap,500}}}}, - {?eh,test_stats,{13,7,{1,19}}}, + {?eh,test_stats,{13,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc2}}, {?eh,tc_done,{cfg_error_12_SUITE,tc2,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{14,7,{1,19}}}, + {?eh,test_stats,{14,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc3}}, {?eh,tc_done,{cfg_error_12_SUITE,tc3,ok}}, - {?eh,test_stats,{15,7,{1,19}}}, + {?eh,test_stats,{15,8,{0,19}}}, {?eh,tc_start,{cfg_error_12_SUITE,tc4}}, {?eh,tc_done,{cfg_error_12_SUITE,tc4,{failed, {cfg_error_12_SUITE,end_per_testcase, {timetrap_timeout,500}}}}}, - {?eh,test_stats,{16,7,{1,19}}}, + {?eh,test_stats,{16,8,{0,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_13_SUITE,tc1}}, {?eh,tc_done,{cfg_error_13_SUITE,tc1,ok}}, - {?eh,test_stats,{17,7,{1,19}}}, + {?eh,test_stats,{17,8,{0,19}}}, {?eh,tc_start,{cfg_error_13_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_13_SUITE,end_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,init_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,init_per_suite,ok}}, {?eh,tc_start,{cfg_error_14_SUITE,tc1}}, {?eh,tc_done,{cfg_error_14_SUITE,tc1,ok}}, - {?eh,test_stats,{18,7,{1,19}}}, + {?eh,test_stats,{18,8,{0,19}}}, {?eh,tc_start,{cfg_error_14_SUITE,end_per_suite}}, {?eh,tc_done,{cfg_error_14_SUITE,end_per_suite, {comment, @@ -1102,7 +1119,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{0,11,{0,2}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g4}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}], @@ -1117,7 +1134,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{0,11,{0,4}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g5}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}], @@ -1132,7 +1149,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}, {?eh,test_stats,{0,11,{0,6}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,g6}, {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}], @@ -1285,7 +1302,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{4,26,{0,8}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg4}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}]}, @@ -1301,7 +1318,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}, {?eh,test_stats,{4,26,{0,10}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg5}, {failed,{timetrap_8_SUITE,init_per_group, {user_timetrap_error,{kaboom,'_'}}}}}}]}, @@ -1317,7 +1334,7 @@ test_events(timetrap_fun_group) -> {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}, {?eh,test_stats,{4,26,{0,12}}}, - {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {?eh,tc_auto_skip,{timetrap_8_SUITE,{end_per_group,pg6}, {failed,{timetrap_8_SUITE,init_per_group, {timetrap_timeout,'_'}}}}}]}, @@ -1498,4 +1515,42 @@ test_events(config_restored) -> {?eh,tc_done,{config_restored_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} + ]; + +test_events(config_func_errors) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{config_func_error_1_SUITE,init_per_suite}}, + {?eh,tc_done,{config_func_error_1_SUITE,init_per_suite,ok}}, + + {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, + {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, + {?eh,test_stats,{0,1,{0,0}}}, + + {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, + {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, + {?eh,test_stats,{0,2,{0,0}}}, + + [{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, + {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}], + + [{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, + {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}], + + {?eh,tc_start,{config_func_error_1_SUITE,end_per_suite}}, + {?eh,tc_done,{config_func_error_1_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/config_func_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/config_func_error_1_SUITE.erl new file mode 100644 index 0000000000..f1025213dc --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/config_func_error_1_SUITE.erl @@ -0,0 +1,138 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(config_func_error_1_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,5}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +bad_proc(Config) -> + ct:pal("Bye bye from ~p", [self()]), + %% this call will either generate an exit immediately + %% or return a fun to be executed here + ErrorFun = ct_test_support:random_error(Config), + ct:log("Calling error fun now...", []), + ErrorFun(), + ct:sleep(10000), + ok. + +init_per_testcase(exit_in_iptc, Config) -> + spawn_link(?MODULE, bad_proc, [Config]), + ct:sleep(10000), + Config; +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(exit_in_eptc, Config) -> + spawn_link(?MODULE, bad_proc, [Config]), + ct:sleep(10000), + ok; +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{g1, [], [exit_in_iptc]}, + {g2, [], [exit_in_eptc]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [exit_in_iptc, + exit_in_eptc, + {group, g1}, + {group, g2}]. + +exit_in_iptc(_) -> + ok. + +exit_in_eptc(_) -> + ok. + diff --git a/lib/common_test/test/ct_group_info_SUITE.erl b/lib/common_test/test/ct_group_info_SUITE.erl index c56fa952e8..e7bc5baaa1 100644 --- a/lib/common_test/test/ct_group_info_SUITE.erl +++ b/lib/common_test/test/ct_group_info_SUITE.erl @@ -269,13 +269,19 @@ test_events(timetrap_all) -> {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_timetrap_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_timetrap_1_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{group_timetrap_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_1_SUITE,{end_per_group,g11,[]},ok}}], - + {?eh,tc_done,{group_timetrap_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip, + {group_timetrap_1_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{group_timetrap_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], + + {?eh,tc_start,{group_timetrap_1_SUITE,end_per_suite}}, {?eh,tc_done,{group_timetrap_1_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; @@ -422,13 +428,15 @@ test_events(timetrap_all_no_ips) -> {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_timetrap_2_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_2_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_timetrap_2_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{group_timetrap_2_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_timetrap_2_SUITE,{end_per_group,g11,[]},ok}}], - - {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,tc_done,{group_timetrap_2_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_timetrap_2_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{group_timetrap_2_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,stop_logging,[]} ]; @@ -501,11 +509,13 @@ test_events(timetrap_all_no_ipg) -> {?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_timetrap_3_SUITE}]},ok}}], [{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}, - {?eh,tc_done,{group_timetrap_3_SUITE,t111,{failed,{timetrap_timeout,1000}}}}, - {?eh,test_stats,{0,14,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_timetrap_3_SUITE}]},ok}}], + {?eh,tc_done,{ct_framework, + {init_per_group,g11,[{suite,group_timetrap_3_SUITE}]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_timetrap_3_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{0,13,{0,1}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -539,11 +549,13 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, - {require_failed,{name_in_use,common2_alias,common2}}}}, + {require_failed, + {name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}}, @@ -566,16 +578,18 @@ test_events(require) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -587,12 +601,16 @@ test_events(require) -> {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_1_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}], - + {?eh,tc_done,{group_require_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], + {?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -627,11 +645,11 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g4,[]}, - {skipped,{require_failed,{not_available,common3}}}}}, + {auto_skipped,{require_failed,{not_available,common3}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t41, {require_failed,{not_available,common3}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g4}, {require_failed,{not_available,common3}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g5,[]}}}, @@ -654,17 +672,19 @@ test_events(require_default) -> [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_1_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_1_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_1_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_1_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g9,[]},ok}}], @@ -676,11 +696,15 @@ test_events(require_default) -> {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g10,[]},ok}}], [{?eh,tc_start,{group_require_1_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_1_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_1_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_1_SUITE,{end_per_group,g11,[]},ok}}], + {?eh,tc_done,{group_require_1_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_1_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,tc_done,{group_require_1_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -714,11 +738,12 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g4,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g4,[]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed, + {name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g5,[]}}}, @@ -741,16 +766,18 @@ test_events(require_no_ips) -> [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g8,[]}}}, {?eh,tc_done,{group_require_2_SUITE, {init_per_group,g8,[]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_2_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{group_require_2_SUITE,end_per_group, + {?eh,tc_auto_skip,{group_require_2_SUITE,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g9,[]},ok}}, {?eh,tc_done,{group_require_2_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed, + {not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g9,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g9,[]},ok}}], @@ -760,13 +787,17 @@ test_events(require_no_ips) -> {?eh,tc_done,{group_require_2_SUITE,t101,ok}}, {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g10,[]}}}, {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g10,[]},ok}}], - + [{?eh,tc_start,{group_require_2_SUITE,{init_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_2_SUITE,{init_per_group,g11,[]},ok}}, - {?eh,tc_done,{group_require_2_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{group_require_2_SUITE,{end_per_group,g11,[]}}}, - {?eh,tc_done,{group_require_2_SUITE,{end_per_group,g11,[]},ok}}], + {?eh,tc_done,{group_require_2_SUITE, + {init_per_group,g11,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_2_SUITE,t111, + {group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{group_require_2_SUITE, + {end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -799,11 +830,11 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g4,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, + {auto_skipped,{require_failed,{name_in_use,common2_alias,common2}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t41, {require_failed,{name_in_use,common2_alias,common2}}}}, {?eh,test_stats,{4,0,{0,1}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4}, {require_failed,{name_in_use,common2_alias,common2}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g5,[{suite,group_require_3_SUITE}]}}}, @@ -825,16 +856,16 @@ test_events(require_no_ipg) -> [{?eh,tc_start,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g8,[{suite,group_require_3_SUITE}]}, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,tc_auto_skip,{group_require_3_SUITE,t81, {require_failed,{not_available,non_existing}}}}, {?eh,test_stats,{8,0,{0,2}}}, - {?eh,tc_auto_skip,{ct_framework,end_per_group, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g8}, {require_failed,{not_available,non_existing}}}}], [{?eh,tc_start,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{init_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}, {?eh,tc_done,{group_require_3_SUITE,t91, - {skipped,{require_failed,{not_available,non_existing}}}}}, + {auto_skipped,{require_failed,{not_available,non_existing}}}}}, {?eh,test_stats,{8,0,{0,3}}}, {?eh,tc_start,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g9,[{suite,group_require_3_SUITE}]},ok}}], @@ -844,13 +875,14 @@ test_events(require_no_ipg) -> {?eh,tc_done,{group_require_3_SUITE,t101,ok}}, {?eh,tc_start,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g10,[{suite,group_require_3_SUITE}]},ok}}], - + [{?eh,tc_start,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}, - {?eh,tc_done,{group_require_3_SUITE,t111,ok}}, - {?eh,test_stats,{10,0,{0,3}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g11,[{suite,group_require_3_SUITE}]},ok}}], + {?eh,tc_done,{ct_framework,{init_per_group,g11,[{suite,group_require_3_SUITE}]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip,{group_require_3_SUITE,t111,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{9,0,{0,4}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g11}, + {group0_failed,bad_return_value}}}], {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index 1b2ad12e2f..8b0de98709 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -171,16 +171,20 @@ test_events(missing_conf) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,start_info,{1,1,2}}, - {?eh,tc_start,{ct_framework,{init_per_group,group1,[]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,group1,[]},ok}}, + {?eh,tc_start,{ct_framework,{init_per_group,group1, + [{suite,missing_conf_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,group1, + [{suite,missing_conf_SUITE}]},ok}}, {?eh,tc_start,{missing_conf_SUITE,tc1}}, {?eh,tc_done,{missing_conf_SUITE,tc1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{missing_conf_SUITE,tc2}}, {?eh,tc_done,{missing_conf_SUITE,tc2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,group1,[]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,group1,[]},ok}}, + {?eh,tc_start,{ct_framework,{end_per_group,group1, + [{suite,missing_conf_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,group1, + [{suite,missing_conf_SUITE}]},ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 596bfe3ff0..b5855da9df 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -80,7 +80,7 @@ all(suite) -> scope_per_suite_state_cth, scope_per_group_state_cth, scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, - fail_post_suite_cth, skip_pre_suite_cth, + fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, state_update_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, @@ -181,6 +181,10 @@ skip_pre_suite_cth(Config) when is_list(Config) -> do_test(skip_pre_suite_cth, "ct_cth_empty_SUITE.erl", [skip_pre_suite_cth],Config). +skip_pre_end_cth(Config) when is_list(Config) -> + do_test(skip_pre_end_cth, "ct_scope_per_group_cth_SUITE.erl", + [skip_pre_end_cth],Config). + skip_post_suite_cth(Config) when is_list(Config) -> do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl", [skip_post_suite_cth],Config). @@ -749,17 +753,53 @@ test_events(skip_pre_suite_cth) -> {?eh,cth,{'_',on_tc_skip, [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, {?eh,stop_logging,[]} ]; +test_events(skip_pre_end_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,init_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, + {?eh,cth,{'_',id,[[]]}}, + {?eh,cth,{'_',init,['_',[]]}}, + {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, + {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, + + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, + {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}, + {skipped,"Test skip"}}}], + {?eh,cth,{'_',on_tc_skip,[end_per_group, + {tc_user_skip,{skipped,"Test skip"}}, + []]}}, + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, + {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite, + {skipped,"Test skip"}}}, + {?eh,cth,{'_',on_tc_skip,[end_per_suite, + {tc_user_skip,{skipped,"Test skip"}}, + []]}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{'_',terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(skip_post_suite_cth) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -773,11 +813,10 @@ test_events(skip_post_suite_cth) -> {?eh,cth,{'_',on_tc_skip, [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, @@ -1219,11 +1258,15 @@ test_events(cth_log) -> {?eh,tc_start,{cth_log_SUITE,init_per_suite}}, {parallel, - [{?eh,tc_start,{ct_framework,{init_per_group,g1,[parallel]}}}, - {?eh,tc_done,{ct_framework,{init_per_group,g1,[parallel]},ok}}, + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,cth_log_SUITE},parallel]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,cth_log_SUITE},parallel]},ok}}, {?eh,test_stats,{30,0,{0,0}}}, - {?eh,tc_start,{ct_framework,{end_per_group,g1,[parallel]}}}, - {?eh,tc_done,{ct_framework,{end_per_group,g1,[parallel]},ok}}]}, + {?eh,tc_start,{ct_framework,{end_per_group,g1, + [{suite,cth_log_SUITE},parallel]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,g1, + [{suite,cth_log_SUITE},parallel]},ok}}]}, {?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl new file mode 100644 index 0000000000..ece4eb2247 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl @@ -0,0 +1,74 @@ +%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(skip_pre_end_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+
+%% CT Hooks
+-compile(export_all).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State),
+ {{skip, "Test skip"}, State}.
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Group,Config,State) ->
+ empty_cth:pre_init_per_group(Group,Config,State).
+
+post_init_per_group(Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Group,Config,Return,State).
+
+pre_end_per_group(Group,Config,State) ->
+ empty_cth:pre_end_per_group(Group,Config,State),
+ {{skip, "Test skip"}, State}.
+
+post_end_per_group(Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Group,Config,Return,State).
+
+pre_init_per_testcase(TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(TC,Config,State).
+
+post_end_per_testcase(TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(TC,Config,Return,State).
+
+on_tc_fail(TC, Reason, State) ->
+ empty_cth:on_tc_fail(TC,Reason,State).
+
+on_tc_skip(TC, Reason, State) ->
+ empty_cth:on_tc_skip(TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 6ee7fdd6f6..2bcfeeec0c 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -84,7 +84,9 @@ all() -> no_host, no_port, invalid_opt, + timeout_close_session, get, + timeout_get, get_xpath, get_config, get_config_xpath, @@ -143,11 +145,10 @@ init_per_suite(Config) -> end. end_per_suite(Config) -> - PrivDir = ?config(priv_dir, Config), ?NS:stop(?config(server,Config)), ssh:stop(), crypto:stop(), - remove_id_keys(PrivDir), + remove_id_keys(Config), Config. hello(Config) -> @@ -344,6 +345,15 @@ invalid_opt(Config) -> {error,{invalid_option,{some_other_opt,true}}} = ct_netconfc:open(Opts2), ok. +timeout_close_session(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Client} = open_success(DataDir), + ?NS:expect('close-session'), + true = erlang:is_process_alive(Client), + {error,timeout} = ct_netconfc:close_session(Client,1000), + false = erlang:is_process_alive(Client), + ok. + get(Config) -> DataDir = ?config(data_dir,Config), {ok,Client} = open_success(DataDir), @@ -354,6 +364,15 @@ get(Config) -> ?ok = ct_netconfc:close_session(Client), ok. +timeout_get(Config) -> + DataDir = ?config(data_dir,Config), + {ok,Client} = open_success(DataDir), + ?NS:expect('get'), + {error,timeout} = ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]},1000), + ?NS:expect_do_reply('close-session',close,ok), + ?ok = ct_netconfc:close_session(Client), + ok. + get_xpath(Config) -> DataDir = ?config(data_dir,Config), {ok,Client} = open_success(DataDir), @@ -698,7 +717,7 @@ timeout_receive_chunked_data(Config) -> ?ok = ct_netconfc:close_session(Client), ok. -%% Same as receive_chunked_data, but timeout waiting for last part. +%% Same as receive_chunked_data, but close while waiting for last part. close_while_waiting_for_chunked_data(Config) -> DataDir = ?config(data_dir,Config), {ok,Client} = open_success(DataDir), diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 09217f60a3..fb0734d48e 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -98,8 +98,9 @@ start(Dir) -> %% Stop the netconf server stop(Pid) -> - Pid ! {stop,self()}, - receive stopped -> ok end. + Ref = erlang:monitor(process,Pid), + Pid ! stop, + receive {'DOWN',Ref,process,Pid,_} -> ok end. %% Set the session id for the hello message. %% If this is not called prior to starting the session, no hello @@ -177,9 +178,9 @@ init_server(Dir) -> loop(Daemon) -> receive - {stop,From} -> + stop -> ssh:stop_daemon(Daemon), - From ! stopped; + ok; {table_trans,Fun,Args,From} -> %% Simple transaction mechanism for ets table R = apply(Fun,Args), diff --git a/lib/common_test/test/ct_repeat_1_SUITE.erl b/lib/common_test/test/ct_repeat_1_SUITE.erl index 090002d0c2..98eaa28763 100644 --- a/lib/common_test/test/ct_repeat_1_SUITE.erl +++ b/lib/common_test/test/ct_repeat_1_SUITE.erl @@ -229,7 +229,7 @@ test_events(repeat_cs_and_grs) -> {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}, {?eh,test_stats,{3,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,test_stats,{4,1,{0,1}}}, @@ -251,7 +251,7 @@ test_events(repeat_cs_and_grs) -> {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}, {?eh,test_stats,{7,2,{0,2}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,test_stats,{8,2,{0,2}}}, @@ -311,7 +311,7 @@ test_events(repeat_seq) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{7,2,{0,5}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed, {repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], @@ -410,7 +410,7 @@ test_events(repeat_gr_until_any_ok) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{1,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,test_stats,{1,2,{0,1}}}, @@ -452,7 +452,7 @@ test_events(repeat_gr_until_any_ok) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{5,6,{0,3}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE, @@ -469,7 +469,7 @@ test_events(repeat_gr_until_any_ok) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{7,7,{0,4}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE, @@ -770,7 +770,7 @@ test_events(repeat_gr_until_any_fail) -> {'EXIT',failing_this_time}}}}}, {?eh,test_stats,{14,0,{0,1}}}, {?eh,tc_auto_skip, - {repeat_1_SUITE,end_per_group, + {repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed, {repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], @@ -967,7 +967,7 @@ test_events(repeat_gr_until_all_ok) -> {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}, {?eh,test_stats,{7,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init_then_ok}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_done,{repeat_1_SUITE,tc_ok_1,ok}}, @@ -1083,7 +1083,7 @@ test_events(repeat_gr_until_all_fail) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{0,1,{0,1}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1,ok}}, @@ -1102,7 +1102,7 @@ test_events(repeat_gr_until_all_fail) -> {init_per_group,gr_fail_init,[]}, {failed,{error,fails_on_purpose}}}}, {?eh,test_stats,{2,2,{0,2}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',fails_on_purpose}}}}}], {?eh,tc_done,{repeat_1_SUITE,tc_ok_then_fail_1, @@ -1134,7 +1134,7 @@ test_events(repeat_gr_until_all_fail) -> {init_per_group,gr_ok_then_fail_init,[]}, {failed,{error,failing_this_time}}}}, {?eh,test_stats,{4,4,{0,3}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_start,{repeat_1_SUITE,tc_fail_1}}, @@ -1293,7 +1293,7 @@ test_events(repeat_seq_until_any_fail) -> {init_per_group,gr_ok_then_fail_init,[]}, {failed,{error,failing_this_time}}}}, {?eh,test_stats,{24,1,{0,5}}}, - {?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_auto_skip,{repeat_1_SUITE,tc_ok_1, @@ -1516,7 +1516,7 @@ test_events(repeat_shuffled_seq_until_any_fail) -> {init_per_group,repeat_shuffled_seq_until_any_fail_5, [{shuffle,'_'},{repeat_until_any_fail,2}, sequence]},ok}}, - [{?eh,tc_auto_skip,{repeat_1_SUITE,end_per_group, + [{?eh,tc_auto_skip,{repeat_1_SUITE,{end_per_group,gr_ok_then_fail_init}, {failed,{repeat_1_SUITE,init_per_group, {'EXIT',failing_this_time}}}}}], {?eh,tc_start,{repeat_1_SUITE, diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index 35d67a10f2..bb2aba2c5a 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -36,7 +36,7 @@ -define(eh, ct_test_support_eh). -define(skip_reason, "Repeated test stopped by force_stop option"). --define(skipped, {skipped, ?skip_reason}). +-define(skipped, {auto_skipped, ?skip_reason}). %% Timers used in this test. @@ -78,7 +78,7 @@ init_per_suite(Config0) -> {1,0,{0,0}} = ct_test_support:run(ct,run_test,[Opts2],Config), %% Time the shortest testcase to use for offset - {T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]), + {_T0,{1,0,{0,0}}} = timer:tc(ct_test_support,run,[ct,run_test,[Opts1],Config]), %% -2 is to ensure we hit inside the target test case and not after % T = round(T0/1000000)-2, @@ -222,7 +222,7 @@ until_force_stop_skip_rest_group(Config) when is_list(Config) -> fun() -> [_] = ct_test_support:run_ct_run_test( Opts++[{until,until_str(?t3,1,Config)}],Config), - 0 = ct_test_support:run_ct_script_start( + 1 = ct_test_support:run_ct_script_start( Opts++[{until,until_str(?t3,1,Config)}],Config) end, ok = execute(ExecuteFun, @@ -341,19 +341,18 @@ skip_first_tc1(Suite) -> {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{1,0}}}, + {?eh,test_stats,{'_',0,{0,1}}}, {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}}, {?eh,tc_auto_skip,{Suite,tc1,?skip_reason}}, - {?eh,test_stats,{'_',0,{1,1}}}, + {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_auto_skip,{Suite,tc2,?skip_reason}}, - {?eh,test_stats,{'_',0,{1,2}}}, - {?eh,tc_auto_skip,{Suite,end_per_group,?skip_reason}}, + {?eh,test_stats,{'_',0,{0,3}}}, + {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{2,2}}}, + {?eh,test_stats,{'_',0,{0,4}}}, {?eh,tc_start,{Suite,end_per_suite}}, {?eh,tc_done,{Suite,end_per_suite,ok}}]. - skip_tc1_in_group(Suite) -> [{?eh,tc_start,{Suite,init_per_suite}}, {?eh,tc_done,{Suite,init_per_suite,ok}}, @@ -369,10 +368,10 @@ skip_tc1_in_group(Suite) -> {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{1,0}}}, + {?eh,test_stats,{'_',0,{0,1}}}, {?eh,tc_start,{Suite,{end_per_group,g,[]}}}, {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}], {?eh,tc_done,{Suite,tc2,?skipped}}, - {?eh,test_stats,{'_',0,{2,0}}}, + {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_start,{Suite,end_per_suite}}, {?eh,tc_done,{Suite,end_per_suite,ok}}]. diff --git a/lib/common_test/test/ct_sequence_1_SUITE.erl b/lib/common_test/test/ct_sequence_1_SUITE.erl index 5facf90656..8c87236838 100644 --- a/lib/common_test/test/ct_sequence_1_SUITE.erl +++ b/lib/common_test/test/ct_sequence_1_SUITE.erl @@ -212,7 +212,7 @@ test_events(subgroup_init_fail) -> {failed,{subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}, {?eh,test_stats,{0,0,{0,1}}}, - {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init}, {failed,{subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}], {?eh,tc_auto_skip,{subgroups_1_SUITE,ok_tc,{group_result,fail_init,failed}}}, @@ -294,7 +294,7 @@ test_events(case_after_subgroup_fail_init) -> {subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}, {?eh,test_stats,{0,0,{0,1}}}, - {?eh,tc_auto_skip,{subgroups_1_SUITE,end_per_group, + {?eh,tc_auto_skip,{subgroups_1_SUITE,{end_per_group,fail_init}, {failed, {subgroups_1_SUITE,init_per_group, {'EXIT',init_per_group_fails_on_purpose}}}}}], diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl index b8be55f43a..b0a6c839a2 100644 --- a/lib/common_test/test/ct_skip_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE.erl @@ -59,7 +59,7 @@ end_per_testcase(TestCase, Config) -> suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [auto_skip, user_skip]. + [auto_skip, user_skip, testspec_skip]. groups() -> []. @@ -91,7 +91,8 @@ auto_skip(Config) when is_list(Config) -> Join(DataDir, "auto_skip_8_SUITE"), Join(DataDir, "auto_skip_9_SUITE"), Join(DataDir, "auto_skip_10_SUITE"), - Join(DataDir, "auto_skip_11_SUITE") + Join(DataDir, "auto_skip_11_SUITE"), + Join(DataDir, "auto_skip_12_SUITE") ], {Opts,ERPid} = setup({suite,Suites}, Config), @@ -116,7 +117,8 @@ user_skip(Config) when is_list(Config) -> Join(DataDir, "user_skip_2_SUITE"), Join(DataDir, "user_skip_3_SUITE"), Join(DataDir, "user_skip_4_SUITE"), - Join(DataDir, "user_skip_5_SUITE")], + Join(DataDir, "user_skip_5_SUITE"), + Join(DataDir, "user_skip_6_SUITE")], {Opts,ERPid} = setup({suite,Suites}, Config), ok = ct_test_support:run(Opts, Config), @@ -131,8 +133,56 @@ user_skip(Config) when is_list(Config) -> ok = ct_test_support:verify_events(TestEvents, Events, Config). %%%----------------------------------------------------------------- +%%% +testspec_skip(Config) when is_list(Config) -> + TestDir = filename:join(?config(data_dir, Config), + filename:join("skip", "test")), + TestSpec1 = [{suites, TestDir, user_skip_7_SUITE}, + {skip_cases, TestDir, user_skip_7_SUITE, [tc1,tc3], "SKIPPED"}], + + TestSpec2 = [{suites, TestDir, user_skip_7_SUITE}, + {skip_groups, TestDir, user_skip_7_SUITE, ptop1, "SKIPPED"}], + + TestSpec3 = [{suites, TestDir, user_skip_7_SUITE}, + {skip_groups, TestDir, user_skip_7_SUITE, psub1, "SKIPPED"}], + + TestSpec4 = [{suites, TestDir, user_skip_7_SUITE}, + {skip_suites, TestDir, user_skip_7_SUITE, "SKIPPED"}], + + TestSpec5 = [{groups, TestDir, user_skip_6_SUITE, ptop1}, + {skip_groups, TestDir, user_skip_6_SUITE, psub1, "SKIPPED"}], + + {Opts,ERPid} = setup_testspec([{ts1,TestSpec1}, + {ts2,TestSpec2}, + {ts3,TestSpec3}, + {ts4,TestSpec4}, + {ts5,TestSpec5}], Config), + + ok = ct_test_support:run(Opts, Config), + + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(testspec_skip, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(testspec_skip), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- +setup_testspec(TestSpecs, Config) -> + SpecFiles = + [begin SpecFile = filename:join(?config(priv_dir, Config), + atom_to_list(SpecName)++".spec"), + {ok,Dev} = file:open(SpecFile, [write]), + [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec], + file:close(Dev), + SpecFile + end || {SpecName,TestSpec} <- TestSpecs], + setup({spec,SpecFiles}, Config). setup(Test, Config) -> Opts0 = ct_test_support:get_opts(Config), @@ -163,7 +213,7 @@ test_events(auto_skip) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{11,11,34}}, + {?eh,start_info,{12,12,43}}, {?eh,tc_start,{auto_skip_1_SUITE,init_per_suite}}, {?eh,tc_done, @@ -196,8 +246,8 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_3_SUITE,tc1}}, {?eh,tc_done, {auto_skip_3_SUITE,tc1, - {skipped,{failed,{auto_skip_3_SUITE,init_per_testcase, - {{init_per_testcase,tc1,failed},'_'}}}}}}, + {auto_skipped,{failed,{auto_skip_3_SUITE,init_per_testcase, + {'init_per_testcase for tc1 failed','_'}}}}}}, {?eh,test_stats,{0,0,{0,4}}}, {?eh,tc_start,{auto_skip_3_SUITE,tc2}}, {?eh,tc_done,{auto_skip_3_SUITE,tc2,ok}}, @@ -209,7 +259,7 @@ test_events(auto_skip) -> {?eh,tc_done,{auto_skip_4_SUITE,init_per_suite,ok}}, {?eh,tc_start,{auto_skip_4_SUITE,tc1}}, {?eh,tc_done,{auto_skip_4_SUITE,tc1, - {skipped,{failed,{auto_skip_4_SUITE,init_per_testcase, + {auto_skipped,{failed,{auto_skip_4_SUITE,init_per_testcase, {timetrap_timeout,1000}}}}}}, {?eh,test_stats,{1,0,{0,5}}}, {?eh,tc_start,{auto_skip_4_SUITE,tc2}}, @@ -232,8 +282,9 @@ test_events(auto_skip) -> {'EXIT',{group,g1,failed}}}}}}, {?eh,test_stats,{2,0,{0,7}}}, {?eh,tc_auto_skip, - {auto_skip_5_SUITE,end_per_group,{failed,{auto_skip_5_SUITE,init_per_group, - {'EXIT',{group,g1,failed}}}}}}], + {auto_skip_5_SUITE,{end_per_group,g1}, + {failed,{auto_skip_5_SUITE,init_per_group, + {'EXIT',{group,g1,failed}}}}}}], {?eh,tc_start,{auto_skip_5_SUITE,end_per_suite}}, {?eh,tc_done,{auto_skip_5_SUITE,end_per_suite,ok}}, @@ -260,8 +311,9 @@ test_events(auto_skip) -> {'EXIT',{group,g1,failed}}}}}}, {?eh,test_stats,{2,0,{0,11}}}, {?eh,tc_auto_skip, - {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group, - {'EXIT',{group,g1,failed}}}}}}], + {auto_skip_6_SUITE,{end_per_group,g1}, + {failed,{auto_skip_6_SUITE,init_per_group, + {'EXIT',{group,g1,failed}}}}}}], [{?eh,tc_start,{auto_skip_6_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{auto_skip_6_SUITE,{init_per_group,g3,[]},ok}}, @@ -280,8 +332,9 @@ test_events(auto_skip) -> {'EXIT',{group,g4,failed}}}}}}, {?eh,test_stats,{3,0,{0,13}}}, {?eh,tc_auto_skip, - {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group, - {'EXIT',{group,g4,failed}}}}}}], + {auto_skip_6_SUITE,{end_per_group,g4}, + {failed,{auto_skip_6_SUITE,init_per_group, + {'EXIT',{group,g4,failed}}}}}}], {?eh,tc_start,{auto_skip_6_SUITE,tc2}}, {?eh,tc_done,{auto_skip_6_SUITE,tc2,ok}}, {?eh,test_stats,{4,0,{0,13}}}, @@ -320,7 +373,7 @@ test_events(auto_skip) -> {?eh,test_stats,{5,0,{0,17}}}, {?eh,tc_start,{auto_skip_9_SUITE,tc2}}, {?eh,tc_done,{auto_skip_9_SUITE,tc2, - {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}}, + {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}}, {?eh,test_stats,{5,0,{0,18}}}, [{?eh,tc_start,{auto_skip_9_SUITE,{init_per_group,g1,[]}}}, @@ -343,7 +396,7 @@ test_events(auto_skip) -> {?eh,tc_done,{auto_skip_9_SUITE,{init_per_group,g3,[]},ok}}, {?eh,tc_start,{auto_skip_9_SUITE,tc5}}, {?eh,tc_done,{auto_skip_9_SUITE,tc5, - {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}}, + {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}}, {?eh,test_stats,{7,0,{0,19}}}, {?eh,tc_start,{auto_skip_9_SUITE,{end_per_group,g3,[]}}}, {?eh,tc_done,{auto_skip_9_SUITE,{end_per_group,g3,[]},ok}}], @@ -363,7 +416,7 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_9_SUITE,tc8}}, {?eh,tc_done, {auto_skip_9_SUITE,tc8, - {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase, + {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase, {{badmatch,undefined},'_'}}}}}}, {?eh,tc_start, {auto_skip_9_SUITE,{end_per_group,g5,[parallel]}}}, @@ -383,26 +436,26 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}}, {?eh,tc_done,{auto_skip_10_SUITE,init_per_suite, - {skipped, - {require_failed_in_suite0, - {not_available,undefined_config_variable}}}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,tc1, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {auto_skipped,{require_failed_in_suite0, + {not_available,undefined_config_variable}}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc1, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,test_stats,{9,0,{0,21}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,tc2, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc2, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,test_stats,{9,0,{0,22}}}, - {?eh,tc_auto_skip, - {auto_skip_10_SUITE,end_per_suite, - {require_failed_in_suite0,{not_available,undefined_config_variable}}}}, + {?eh,tc_auto_skip,{auto_skip_10_SUITE,end_per_suite, + {require_failed_in_suite0, + {not_available,undefined_config_variable}}}}, {?eh,tc_start,{auto_skip_11_SUITE,init_per_suite}}, {?eh,tc_done,{auto_skip_11_SUITE,init_per_suite,ok}}, {?eh,tc_start,{auto_skip_11_SUITE,tc1}}, {?eh,tc_done,{auto_skip_11_SUITE,tc1, - {skipped,{require_failed,{not_available,undefined_config_variable}}}}}, + {auto_skipped,{require_failed, + {not_available,undefined_config_variable}}}}}, {?eh,test_stats,{9,0,{0,23}}}, {?eh,tc_start,{auto_skip_11_SUITE,tc2}}, {?eh,tc_done,{auto_skip_11_SUITE,tc2,ok}}, @@ -421,7 +474,8 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_11_SUITE,tc3}}, {?eh,tc_done, {auto_skip_11_SUITE,tc3, - {skipped,{require_failed,{not_available,undefined_config_variable}}}}}, + {auto_skipped,{require_failed, + {not_available,undefined_config_variable}}}}}, {?eh,test_stats,{10,0,{0,24}}}, {?eh,tc_start, {auto_skip_11_SUITE,{end_per_group,g2,[parallel]}}}, @@ -434,6 +488,75 @@ test_events(auto_skip) -> {?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}}, {?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}}, + + {?eh,tc_start,{auto_skip_12_SUITE,init_per_suite}}, + {?eh,tc_done,{auto_skip_12_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g1}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,25}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,26}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,27}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1}, + {require_failed,{not_available,unknown_variable_g1}}}}], + + [{?eh,tc_start,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g1, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g1}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,28}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,29}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g1}}}}, + {?eh,test_stats,{10,0,{0,30}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1}, + {require_failed,{not_available,unknown_variable_g1}}}}], + + [{?eh,tc_start,{ct_framework,{init_per_group,g3, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g3, + [{suite,auto_skip_12_SUITE}]},ok}}, + {?eh,tc_start,{auto_skip_12_SUITE,tc1}}, + {?eh,tc_done,{auto_skip_12_SUITE,tc1,ok}}, + {?eh,test_stats,{11,0,{0,30}}}, + {?eh,tc_start,{auto_skip_12_SUITE,tc2}}, + {?eh,tc_done,{auto_skip_12_SUITE,tc2,ok}}, + {?eh,test_stats,{12,0,{0,30}}}, + [{?eh,tc_start,{ct_framework,{init_per_group,g4, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{init_per_group,g4, + [{suite,auto_skip_12_SUITE}]}, + {auto_skipped, + {require_failed,{not_available,unknown_variable_g4}}}}}, + {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3, + {require_failed,{not_available,unknown_variable_g4}}}}, + {?eh,test_stats,{12,0,{0,31}}}, + {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4}, + {require_failed,{not_available,unknown_variable_g4}}}}], + + {?eh,tc_start,{ct_framework,{end_per_group,g3, + [{suite,auto_skip_12_SUITE}]}}}, + {?eh,tc_done,{ct_framework,{end_per_group,g3, + [{suite,auto_skip_12_SUITE}]},ok}}], + + {?eh,tc_start,{auto_skip_12_SUITE,end_per_suite}}, + {?eh,tc_done,{auto_skip_12_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} @@ -442,46 +565,46 @@ test_events(auto_skip) -> test_events(user_skip) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{5,5,27}}, + {?eh,start_info,{6,6,35}}, {?eh,tc_start,{user_skip_1_SUITE,init_per_suite}}, {?eh,tc_done, {user_skip_1_SUITE,init_per_suite,{skipped,"Whole suite skipped"}}}, - {?eh,tc_auto_skip, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc1,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,1}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc2,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,2}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc3,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,3}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,tc4,"Whole suite skipped"}}, - {?eh,test_stats,{0,0,{0,4}}}, - {?eh,tc_auto_skip, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_user_skip, {user_skip_1_SUITE,end_per_suite,"Whole suite skipped"}}, {?eh,tc_start,{user_skip_2_SUITE,init_per_suite}}, {?eh,tc_done,{user_skip_2_SUITE,init_per_suite,ok}}, {?eh,tc_start,{user_skip_2_SUITE,tc1}}, {?eh,tc_done,{user_skip_2_SUITE,tc1,{skipped,{tc1,skipped}}}}, - {?eh,test_stats,{0,0,{1,4}}}, + {?eh,test_stats,{0,0,{5,0}}}, [{?eh,tc_start,{user_skip_2_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_2_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{user_skip_2_SUITE,tc2}}, {?eh,tc_done,{user_skip_2_SUITE,tc2,ok}}, - {?eh,test_stats,{1,0,{1,4}}}, + {?eh,test_stats,{1,0,{5,0}}}, {?eh,tc_start,{user_skip_2_SUITE,tc3}}, {?eh,tc_done,{user_skip_2_SUITE,tc3,{skipped,{tc3,skipped}}}}, - {?eh,test_stats,{1,0,{2,4}}}, + {?eh,test_stats,{1,0,{6,0}}}, {?eh,tc_start,{user_skip_2_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_2_SUITE,{end_per_group,g1,[]},ok}}], {?eh,tc_start,{user_skip_2_SUITE,tc4}}, {?eh,tc_done,{user_skip_2_SUITE,tc4,ok}}, - {?eh,test_stats,{2,0,{2,4}}}, + {?eh,test_stats,{2,0,{6,0}}}, {?eh,tc_start,{user_skip_2_SUITE,end_per_suite}}, {?eh,tc_done,{user_skip_2_SUITE,end_per_suite,ok}}, @@ -489,16 +612,16 @@ test_events(user_skip) -> {?eh,tc_done,{user_skip_3_SUITE,init_per_suite,ok}}, {?eh,tc_start,{user_skip_3_SUITE,tc1}}, {?eh,tc_done,{user_skip_3_SUITE,tc1,{skipped,"Test case skipped"}}}, - {?eh,test_stats,{2,0,{3,4}}}, + {?eh,test_stats,{2,0,{7,0}}}, [{?eh,tc_start,{user_skip_3_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_3_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{user_skip_3_SUITE,tc2}}, {?eh,tc_done,{user_skip_3_SUITE,tc2,ok}}, - {?eh,test_stats,{3,0,{3,4}}}, + {?eh,test_stats,{3,0,{7,0}}}, {?eh,tc_start,{user_skip_3_SUITE,tc3}}, {?eh,tc_done,{user_skip_3_SUITE,tc3,{skipped,"Test case skipped"}}}, - {?eh,test_stats,{3,0,{4,4}}}, + {?eh,test_stats,{3,0,{8,0}}}, {?eh,tc_start,{user_skip_3_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_3_SUITE,{end_per_group,g1,[]},ok}}], @@ -506,7 +629,7 @@ test_events(user_skip) -> {?eh,tc_done,{user_skip_3_SUITE,tc4, {skipped,{proc_info,{{current_function,{user_skip_3_SUITE,tc4,1}}, {initial_call,{erlang,apply,2}}}}}}}, - {?eh,test_stats,{3,0,{5,4}}}, + {?eh,test_stats,{3,0,{9,0}}}, {?eh,tc_start,{user_skip_3_SUITE,end_per_suite}}, {?eh,tc_done,{user_skip_3_SUITE,end_per_suite,ok}}, @@ -515,63 +638,219 @@ test_events(user_skip) -> [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc1,"Group skipped"}}, - {?eh,test_stats,{3,0,{5,5}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc2,"Group skipped"}}, - {?eh,test_stats,{3,0,{5,6}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}}, + {?eh,test_stats,{3,0,{10,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}}, + {?eh,test_stats,{3,0,{11,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g1},"Group skipped"}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_start,{user_skip_4_SUITE,tc3}}, {?eh,tc_done,{user_skip_4_SUITE,tc3,ok}}, - {?eh,test_stats,{4,0,{5,6}}}, + {?eh,test_stats,{4,0,{11,0}}}, {?eh,tc_start,{user_skip_4_SUITE,tc4}}, {?eh,tc_done,{user_skip_4_SUITE,tc4,ok}}, - {?eh,test_stats,{5,0,{5,6}}}, + {?eh,test_stats,{5,0,{11,0}}}, {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g2,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g2,[]},ok}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc5,"Group skipped"}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc6,"Group skipped"}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc7,"Group skipped"}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc8,"Group skipped"}}, - {?eh,test_stats,{5,0,{5,10}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}}, + {?eh,test_stats,{5,0,{15,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g3},"Group skipped"}}], [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_start,{user_skip_4_SUITE,tc9}}, {?eh,tc_done,{user_skip_4_SUITE,tc9,ok}}, - {?eh,test_stats,{6,0,{5,10}}}, + {?eh,test_stats,{6,0,{15,0}}}, [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc10,"Group skipped"}}, - {?eh,test_stats,{6,0,{5,11}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,tc11,"Group skipped"}}, - {?eh,test_stats,{6,0,{5,12}}}, - {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}], + {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}}, + {?eh,test_stats,{6,0,{16,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}}, + {?eh,test_stats,{6,0,{17,0}}}, + {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g6},"Group skipped"}}], {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}}, {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}], {?eh,tc_start,{user_skip_4_SUITE,end_per_suite}}, {?eh,tc_done,{user_skip_4_SUITE,end_per_suite,ok}}, - {ct_test_support_eh,tc_start,{user_skip_5_SUITE,init_per_suite}}, + {?eh,tc_start,{user_skip_5_SUITE,init_per_suite}}, {?eh,tc_done,{user_skip_5_SUITE,init_per_suite, {skipped,{bad,'Whole suite skipped'}}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,13}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,14}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,15}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}}, - {?eh,test_stats,{6,0,{5,16}}}, - {?eh,tc_auto_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{18,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{19,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{20,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}}, + {?eh,test_stats,{6,0,{21,0}}}, + {?eh,tc_user_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}}, + + {parallel, + [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}}, + {?eh,tc_done,{user_skip_6_SUITE, + {init_per_group,ptop1,[parallel]}, + {skipped,"Top group skipped"}}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Top group skipped"}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Top group skipped"}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,ptop1}, + "Top group skipped"}}]}, + + {parallel, + [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]}}}, + {?eh,tc_done,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]},ok}}, + {?eh,tc_start,{user_skip_6_SUITE,tc1}}, + {?eh,tc_done,{user_skip_6_SUITE,tc1,ok}}, + + {parallel, + [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,psub2,[parallel]}}}, + {?eh,tc_done,{user_skip_6_SUITE, + {init_per_group,psub2,[parallel]}, + {skipped,"Sub group skipped"}}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Sub group skipped"}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Sub group skipped"}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,psub2}, + "Sub group skipped"}}]}, + + {?eh,tc_start,{user_skip_6_SUITE,tc2}}, + {?eh,tc_done,{user_skip_6_SUITE,tc2,ok}}, + {?eh,test_stats,{8,0,{27,0}}}, + {?eh,tc_start,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]}}}, + {?eh,tc_done,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]},ok}}]}, + + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + +test_events(testspec_skip) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {parallel, + [{?eh,tc_start, + {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}}, + {?eh,tc_done, + {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}}, + {?eh,test_stats,{0,0,{1,0}}}, + {parallel, + [{?eh,tc_start, + {user_skip_7_SUITE,{init_per_group,psub1,[parallel]}}}, + {?eh,tc_done, + {user_skip_7_SUITE,{init_per_group,psub1,[parallel]},ok}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}}, + {?eh,tc_start,{user_skip_7_SUITE,tc4}}, + {?eh,tc_done,{user_skip_7_SUITE,tc4,ok}}, + {?eh,test_stats,{1,0,{2,0}}}, + {?eh,tc_start, + {user_skip_7_SUITE,{end_per_group,psub1,[parallel]}}}, + {?eh,tc_done, + {user_skip_7_SUITE,{end_per_group,psub1,[parallel]},ok}}]}, + {?eh,tc_start,{user_skip_7_SUITE,tc2}}, + {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_start, + {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}}, + {?eh,tc_done, + {user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,{init_per_group,ptop1},"SKIPPED"}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}}, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,ptop1},"SKIPPED"}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {parallel, + [{?eh,tc_start, + {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}}, + {?eh,tc_done, + {user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}}, + {?eh,tc_user_skip, + {user_skip_7_SUITE,{init_per_group,psub1},"SKIPPED"}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,psub1},"SKIPPED"}}, + {?eh,tc_start,{user_skip_7_SUITE,tc1}}, + {?eh,tc_done,{user_skip_7_SUITE,tc1,ok}}, + {?eh,tc_start,{user_skip_7_SUITE,tc2}}, + {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_start,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}}, + {?eh,tc_done,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,0}}, + {?eh,tc_user_skip,{user_skip_7_SUITE,all,"SKIPPED"}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]}, + + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,4}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {parallel, + [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}}, + {?eh,tc_done,{user_skip_6_SUITE, + {init_per_group,ptop1,[parallel]}, + {skipped,"Top group skipped"}}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"SKIPPED"}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"SKIPPED"}}, + {?eh,test_stats,{0,0,{3,0}}}, + {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_user_skip, + {user_skip_6_SUITE,{end_per_group,ptop1},"Top group skipped"}}]}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]. + + diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl new file mode 100644 index 0000000000..a168afd386 --- /dev/null +++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_12_SUITE.erl @@ -0,0 +1,121 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(auto_skip_12_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + []. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: group(Name) -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +group(g1) -> + [{require,unknown_variable_g1}]; +group(g4) -> + [{require,unknown_variable_g4}]; +group(_) -> + []. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{g1,[],[tc1,tc2,{g2,[],[tc3]}]}, + {g1,[],[tc1,tc2,{g2,[],[tc3]}]}, + {g3,[],[tc1,tc2,{g4,[],[tc3]}]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [{group,g1}, + {group,g3}]. + +%%-------------------------------------------------------------------- +%% Function: TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%%-------------------------------------------------------------------- +tc1(_Config) -> + ok. + +tc2(_Config) -> + ok. + +tc3(_Config) -> + ok. diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl index cb64cb76c5..4ef9f50514 100644 --- a/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl +++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/auto_skip_3_SUITE.erl @@ -72,7 +72,7 @@ end_per_group(_GroupName, _Config) -> %% Reason = term() %%-------------------------------------------------------------------- init_per_testcase(tc1, _Config) -> - exit({init_per_testcase,tc1,failed}); + exit('init_per_testcase for tc1 failed'); init_per_testcase(_TestCase, Config) -> Config. diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl new file mode 100644 index 0000000000..5f0e5db6f2 --- /dev/null +++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_6_SUITE.erl @@ -0,0 +1,118 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(user_skip_6_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,30}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(ptop1, Config) -> + {skip,"Top group skipped"}; +init_per_group(psub2, Config) -> + {skip,"Sub group skipped"}; +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]}, + {ptop2,[parallel],[tc1,{psub2,[parallel],[tc3,tc4]},tc2]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [{group,ptop1},{group,ptop2}]. + +%%-------------------------------------------------------------------- +%% Function: TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%%-------------------------------------------------------------------- +tc1(_) -> + ok. + +tc2(_) -> + ok. + +tc3(_) -> + ok. + +tc4(_) -> + ok. diff --git a/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl new file mode 100644 index 0000000000..82ce536a79 --- /dev/null +++ b/lib/common_test/test/ct_skip_SUITE_data/skip/test/user_skip_7_SUITE.erl @@ -0,0 +1,113 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(user_skip_7_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{seconds,30}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_TestCase, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_TestCase, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [{ptop1,[parallel],[tc1,{psub1,[parallel],[tc3,tc4]},tc2]}]. + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [{group,ptop1}]. + +%%-------------------------------------------------------------------- +%% Function: TestCase(Config0) -> +%% ok | exit() | {skip,Reason} | {comment,Comment} | +%% {save_config,Config1} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%% Comment = term() +%%-------------------------------------------------------------------- +tc1(_) -> + ok. + +tc2(_) -> + ok. + +tc3(_) -> + ok. + +tc4(_) -> + ok. diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index b86b47f0a2..c5e44682b0 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -182,7 +182,7 @@ test_events(_) -> {?eh,test_stats,{1,1,{1,0}}}, {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {skipped,{require_failed,'_'}}}}, + {auto_skipped,{require_failed,'_'}}}}, {?eh,test_stats,{1,1,{1,1}}}, [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, @@ -198,7 +198,7 @@ test_events(_) -> {?eh,test_stats,{2,2,{2,1}}}, {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {skipped,{require_failed,'_'}}}}, + {auto_skipped,{require_failed,'_'}}}}, {?eh,test_stats,{2,2,{2,2}}}, {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], @@ -210,7 +210,7 @@ test_events(_) -> {surefire_SUITE,init_per_group, {'EXIT',all_cases_should_be_skipped}}}}}, {?eh,test_stats,{2,2,{2,3}}}, - {?eh,tc_auto_skip,{surefire_SUITE,end_per_group, + {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, {failed, {surefire_SUITE,init_per_group, {'EXIT',all_cases_should_be_skipped}}}}}], @@ -328,6 +328,7 @@ events_to_result([]) -> result(ok) ->[]; result({skipped,_}) -> [s]; +result({auto_skipped,_}) -> [s]; result({failed,_}) -> [f]. %% Using the expected events' last test_stats element to produce the diff --git a/lib/common_test/test/ct_telnet_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE.erl index e2ee207754..acce4eca14 100644 --- a/lib/common_test/test/ct_telnet_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE.erl @@ -46,6 +46,26 @@ %% instance, the tests need to be performed on a separate node (or %% there will be clashes with logging processes etc). %%-------------------------------------------------------------------- +suite() -> [{ct_hooks,[ts_install_cth]}]. + +groups() -> + [{legacy, [], [unix_telnet,own_server,timetrap]}, + {raw, [], [unix_telnet,own_server,timetrap]}, + {html, [], [unix_telnet,own_server]}, + {silent, [], [unix_telnet,own_server]}]. + +all() -> + [ + {group,legacy}, + {group,raw}, + {group,html}, + {group,silent} + ]. + +%%-------------------------------------------------------------------- +%% CONFIG FUNCTIONS +%%-------------------------------------------------------------------- + init_per_suite(Config) -> ct_test_support:init_per_suite(Config). @@ -67,14 +87,6 @@ end_per_testcase(TestCase, Config) -> end, ct_test_support:end_per_testcase(TestCase, Config). -suite() -> [{ct_hooks,[ts_install_cth]}]. - -all() -> - [ - unix_telnet, - own_server, - timetrap - ]. %%-------------------------------------------------------------------- %% TEST CASES @@ -82,27 +94,43 @@ all() -> %%%----------------------------------------------------------------- %%% -unix_telnet(Config) when is_list(Config) -> - all_tests_in_suite(unix_telnet,"ct_telnet_basic_SUITE","telnet.cfg",Config). +unix_telnet(Config) -> + CfgFile = "telnet.unix_telnet." ++ + atom_to_list(groupname(Config)) ++ ".cfg", + all_tests_in_suite(unix_telnet,"ct_telnet_basic_SUITE",CfgFile,Config). own_server(Config) -> + CfgFile = "telnet.own_server." ++ + atom_to_list(groupname(Config)) ++ ".cfg", all_tests_in_suite(own_server,"ct_telnet_own_server_SUITE", - "telnet2.cfg",Config). + CfgFile,Config). timetrap(Config) -> + CfgFile = "telnet.timetrap." ++ + atom_to_list(groupname(Config)) ++ ".cfg", all_tests_in_suite(timetrap,"ct_telnet_timetrap_SUITE", - "telnet3.cfg",Config). + CfgFile,Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- +groupname(Config) -> + case proplists:get_value(tc_group_properties, Config) of + undefined -> + undefined; + TGP -> + proplists:get_value(name, TGP) + end. + all_tests_in_suite(TestCase, SuiteName, CfgFileName, Config) -> + PrivDir = ?config(priv_dir, Config), DataDir = ?config(data_dir, Config), Suite = filename:join(DataDir, SuiteName), - CfgFile = filename:join(DataDir, CfgFileName), - Cfg = telnet_config(TestCase), - ok = file:write_file(CfgFile, io_lib:write(Cfg) ++ "."), + CfgFile = filename:join(PrivDir, CfgFileName), + Cfg = telnet_config(TestCase, groupname(Config)), + Txt = lists:flatten([lists:flatten(io_lib:write(C))++".\n" || C <- Cfg]), + ok = file:write_file(CfgFile, Txt), {Opts,ERPid} = setup([{suite,Suite}, {label,TestCase}, {config,CfgFile}], @@ -132,15 +160,36 @@ execute(Name, Opts, ERPid, Config) -> reformat(Events, EH) -> ct_test_support:reformat(Events, EH). - -telnet_config(unix_telnet) -> - {unix, ct:get_config(unix)}; -telnet_config(_) -> - {unix,[{telnet,"localhost"}, - {port, ?erl_telnet_server_port}, - {username,?erl_telnet_server_user}, - {password,?erl_telnet_server_pwd}, - {keep_alive,true}]}. +telnet_config(_, undefined) -> + []; +telnet_config(unix_telnet, legacy) -> + [{unix, ct:get_config(unix)}, + {ct_conn_log,[]}]; +%% LogType same as GroupName +telnet_config(unix_telnet, LogType) -> + [{unix, ct:get_config(unix)}, + {ct_conn_log, + [{ct_telnet,[{log_type,LogType}, + {hosts,[telnet_server_conn1, + telnet_server_conn2, + telnet_server_conn3, + telnet_server_conn4]}]}]}]; +telnet_config(_, LogType) -> + [{unix,[{telnet,"localhost"}, + {port, ?erl_telnet_server_port}, + {username,?erl_telnet_server_user}, + {password,?erl_telnet_server_pwd}, + {keep_alive,true}]} | + if LogType == legacy -> + [{ct_conn_log,[]}]; + true -> + [{ct_conn_log, + [{ct_telnet,[{log_type,LogType}, + {hosts,[telnet_server_conn1, + telnet_server_conn2, + telnet_server_conn3, + telnet_server_conn4]}]}]}] + end]. %%%----------------------------------------------------------------- %%% TEST EVENTS @@ -159,14 +208,39 @@ events_to_check(timetrap,_Config) -> all_cases(Suite,Config) -> {module,_} = code:load_abs(filename:join(?config(data_dir,Config), Suite)), - TCs = Suite:all(), + GroupsAndTCs = Suite:all(), + + Terms = + lists:flatmap( + fun({group,G}) -> + {value,{G,Props,GTCs}} = + lists:keysearch(G,1,Suite:groups()), + GTCs1 = [[{?eh,tc_start,{Suite,GTC}}, + {?eh,tc_done,{Suite,GTC,ok}}] || + GTC <- GTCs], + GEvs = [{?eh,tc_start,{Suite,{init_per_group,G,Props}}}, + {?eh,tc_done,{Suite,{init_per_group,G,Props},ok}} | + GTCs1] ++ + [{?eh,tc_start,{Suite,{end_per_group,G,Props}}}, + {?eh,tc_done,{Suite,{end_per_group,G,Props},ok}}], + case lists:member(parallel, Props) of + true -> [{parallel,GEvs}]; + false -> GEvs + end; + (TC) -> + [{?eh,tc_done,{Suite,TC,ok}}] + end, GroupsAndTCs), + code:purge(Suite), code:delete(Suite), + FlatTerms = lists:flatten(Terms), + + ct:log("Verifying with terms:~n~p", [FlatTerms]), + OneTest = - [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ - [{?eh,tc_done,{Suite,TC,ok}} || TC <- TCs] ++ - [{?eh,stop_logging,[]}], + [{?eh,start_logging,{'DEF','RUNDIR'}} | + FlatTerms] ++ [{?eh,stop_logging,[]}], %% 2 tests (ct:run_test + script_start) is default OneTest ++ OneTest. diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_basic_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_basic_SUITE.erl index 914b95f9cf..80616af064 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_basic_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_basic_SUITE.erl @@ -5,45 +5,101 @@ -include_lib("common_test/include/ct.hrl"). +-define(no_of_sessions, 4). +-define(conn_name(N), (list_to_atom("telnet_server_conn"++integer_to_list(N)))). +-define(get_n(Cfg), (proplists:get_value(n, Cfg))). + %%-------------------------------------------------------------------- %% TEST SERVER CALLBACK FUNCTIONS %%-------------------------------------------------------------------- +suite() -> [ + {require,ct_conn_log}, + {ct_hooks, [{cth_conn_log,[]}]} + ]. + +operations() -> + [start_stop, send_and_get, expect, already_closed, + cmd, sendf, close_wrong_type]. + +mult_case(_Case, 0) -> + []; +mult_case(Case, N) -> + [list_to_atom(atom_to_list(Case)++integer_to_list(N)) | + mult_case(Case, N-1)]. + +groups() -> + [{single_connection,[],operations()}, + {multiple_connections,[parallel], + lists:reverse(mult_case(sessions,?no_of_sessions))}]. + +all() -> + [{group,single_connection}, + {group,multiple_connections}]. + init_per_suite(Config) -> + ct:pal("Will use these log hook options: ~p", + [ct:get_config(ct_conn_log,[])]), Config. end_per_suite(_Config) -> ok. +init_per_group(Group, Config) -> + if Group == single_connection -> + ct:require(?conn_name(1),{unix,[telnet]}); + true -> + ok + end, + [{n,1} | Config]. -suite() -> [{require,telnet_temp,{unix,[telnet]}}]. +end_per_group(_GroupName, Config) -> + Config. -all() -> - [start_stop, send_and_get, expect, already_closed, - cmd, sendf, close_wrong_type]. +init_per_testcase(Case, Config) when (Case == sessions1) or + (Case == sessions2) or + (Case == sessions3) or + (Case == sessions4) -> + N = lists:last(atom_to_list(Case))-48, + ct:log("Using connection ~w for session ~w on ~w", + [?conn_name(N),N,self()]), + ct:require(?conn_name(N),{unix,[telnet]}), + [{n,N} | proplists:delete(n,Config)]; +init_per_testcase(Case, Config) -> + ct:log("Testcase ~w using connection ~w", + [Case,?conn_name(?get_n(Config))]), + Config. -groups() -> - []. +end_per_testcase(_Case, _) -> + ok. -init_per_group(_GroupName, Config) -> - Config. +%%%----------------------------------------------------------------- +%%% TEST CASES -end_per_group(_GroupName, Config) -> - Config. +sessions(Config) -> + [apply(?MODULE,Op,[Config]) || Op <- operations()], + ok. + +sessions1(Config) -> sessions(Config). +sessions2(Config) -> sessions(Config). +sessions3(Config) -> sessions(Config). +sessions4(Config) -> sessions(Config). -start_stop(_Config) -> - {ok, Handle} = ct_telnet:open(telnet_temp), +start_stop(Config) -> + ct:log("Opening ~w...", [?conn_name(?get_n(Config))]), + {ok, Handle} = ct_telnet:open(?conn_name(?get_n(Config))), ok = ct_telnet:close(Handle), ok. -send_and_get(_) -> - {ok, Handle} = ct_telnet:open(telnet_temp), + +send_and_get(Config) -> + {ok, Handle} = ct_telnet:open(?conn_name(?get_n(Config))), ok = ct_telnet:send(Handle, "ayt"), {ok, _Data} = ct_telnet:get_data(Handle), ok = ct_telnet:close(Handle), ok. -expect(_) -> - {ok, Handle} = ct_telnet:open(telnet_temp), +expect(Config) -> + {ok, Handle} = ct_telnet:open(?conn_name(?get_n(Config))), ok = ct_telnet:send(Handle, "echo ayt"), ok = case ct_telnet:expect(Handle, ["ayt"]) of {ok, _} -> @@ -54,21 +110,21 @@ expect(_) -> ok = ct_telnet:close(Handle), ok. -already_closed(_) -> - {ok, Handle} = ct_telnet:open(telnet_temp), +already_closed(Config) -> + {ok, Handle} = ct_telnet:open(?conn_name(?get_n(Config))), ok = ct_telnet:close(Handle), {error, already_closed} = ct_telnet:close(Handle), ok. -cmd(_) -> - {ok, Handle} = ct_telnet:open(telnet_temp), +cmd(Config) -> + {ok, Handle} = ct_telnet:open(?conn_name(?get_n(Config))), {ok, _} = ct_telnet:cmd(Handle, "display"), {ok, _} = ct_telnet:cmdf(Handle, "~s ~s", ["set", "bsasdel"]), ok = ct_telnet:close(Handle), ok. -sendf(_) -> - {ok, Handle} = ct_telnet:open(telnet_temp), +sendf(Config) -> + {ok, Handle} = ct_telnet:open(?conn_name(?get_n(Config))), ok = ct_telnet:sendf(Handle, "~s", ["ayt"]), ok = ct_telnet:close(Handle), ok. @@ -76,3 +132,8 @@ sendf(_) -> close_wrong_type(_) -> {error, _} = ct_telnet:close(whatever), ok. + +%%%----------------------------------------------------------------- +%%% HELP FUNCS + + diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl index 3f7c0d68bf..8d142e85a8 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_own_server_SUITE.erl @@ -8,14 +8,12 @@ %% TEST SERVER CALLBACK FUNCTIONS %%-------------------------------------------------------------------- -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - - -suite() -> [{require,erl_telnet_server,{unix,[telnet]}}]. +suite() -> + [ + {require,telnet_server_conn1,{unix,[telnet]}}, + {require,ct_conn_log}, + {ct_hooks, [{cth_conn_log,[]}]} + ]. all() -> [expect, @@ -35,6 +33,14 @@ all() -> groups() -> []. +init_per_suite(Config) -> + ct:pal("Will use these log hook options: ~p", + [ct:get_config(ct_conn_log,[])]), + Config. + +end_per_suite(_Config) -> + ok. + init_per_group(_GroupName, Config) -> Config. @@ -43,7 +49,7 @@ end_per_group(_GroupName, Config) -> %% Simple expect expect(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo ayt"), {ok,["ayt"]} = ct_telnet:expect(Handle, ["ayt"]), ok = ct_telnet:close(Handle), @@ -51,7 +57,7 @@ expect(_) -> %% Expect with repeat option expect_repeat(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_ml xy xy"), {ok,[["xy"],["xy"]],done} = ct_telnet:expect(Handle, ["xy"],[{repeat,2}]), ok = ct_telnet:close(Handle), @@ -59,7 +65,7 @@ expect_repeat(_) -> %% Expect with sequence option expect_sequence(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_ml ab cd ef"), {ok,[["ab"],["cd"],["ef"]]} = ct_telnet:expect(Handle, [["ab"],["cd"],["ef"]], @@ -70,7 +76,7 @@ expect_sequence(_) -> %% Check that expect returns when a prompt is found, even if pattern %% is not matched. expect_error_prompt(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo xxx> yyy"), {error,{prompt,"> "}} = ct_telnet:expect(Handle, ["yyy"]), ok = ct_telnet:close(Handle), @@ -80,7 +86,7 @@ expect_error_prompt(_) -> %% expected pattern is received - as long as not newline or prompt is %% received it will not match. expect_error_timeout(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_no_prompt xxx"), {error,timeout} = ct_telnet:expect(Handle, ["xxx"], [{timeout,1000}]), ok = ct_telnet:close(Handle), @@ -89,7 +95,7 @@ expect_error_timeout(_) -> %% expect with ignore_prompt option should not return even if a prompt %% is found. The pattern after the prompt (here "> ") can be matched. ignore_prompt(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo xxx> yyy"), {ok,["yyy"]} = ct_telnet:expect(Handle, ["yyy"], [ignore_prompt]), ok = ct_telnet:close(Handle), @@ -97,7 +103,7 @@ ignore_prompt(_) -> %% expect with ignore_prompt and repeat options. ignore_prompt_repeat(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_ml yyy> yyy>"), {ok,[["yyy"],["yyy"]],done} = ct_telnet:expect(Handle, ["yyy"], [{repeat,2}, @@ -107,7 +113,7 @@ ignore_prompt_repeat(_) -> %% expect with ignore_prompt and sequence options. ignore_prompt_sequence(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_ml xxx> yyy> zzz> "), {ok,[["xxx"],["yyy"],["zzz"]]} = ct_telnet:expect(Handle, [["xxx"],["yyy"],["zzz"]], @@ -121,7 +127,7 @@ ignore_prompt_sequence(_) -> %% As for expect without the ignore_prompt option, it a newline or a %% prompt is required in order for the pattern to match. ignore_prompt_timeout(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo xxx"), {error,timeout} = ct_telnet:expect(Handle, ["yyy"], [ignore_prompt, {timeout,1000}]), @@ -140,7 +146,7 @@ ignore_prompt_timeout(_) -> %% no_prompt_check option shall match pattern both when prompt is sent %% and when it is not. no_prompt_check(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo xxx"), {ok,["xxx"]} = ct_telnet:expect(Handle, ["xxx"], [no_prompt_check]), ok = ct_telnet:send(Handle, "echo_no_prompt yyy"), @@ -150,7 +156,7 @@ no_prompt_check(_) -> %% no_prompt_check and repeat options no_prompt_check_repeat(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_ml xxx xxx"), {ok,[["xxx"],["xxx"]],done} = ct_telnet:expect(Handle,["xxx"], [{repeat,2}, @@ -164,7 +170,7 @@ no_prompt_check_repeat(_) -> %% no_prompt_check and sequence options no_prompt_check_sequence(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo_ml_no_prompt ab cd ef"), {ok,[["ab"],["cd"],["ef"]]} = ct_telnet:expect(Handle, [["ab"],["cd"],["ef"]], @@ -176,7 +182,7 @@ no_prompt_check_sequence(_) -> %% Check that expect returns after idle timeout when no_prompt_check %% option is used. no_prompt_check_timeout(_) -> - {ok, Handle} = ct_telnet:open(erl_telnet_server), + {ok, Handle} = ct_telnet:open(telnet_server_conn1), ok = ct_telnet:send(Handle, "echo xxx"), {error,timeout} = ct_telnet:expect(Handle, ["yyy"], [no_prompt_check, {timeout,1000}]), diff --git a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_timetrap_SUITE.erl b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_timetrap_SUITE.erl index f274fb9112..c45a4f0984 100644 --- a/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_timetrap_SUITE.erl +++ b/lib/common_test/test/ct_telnet_SUITE_data/ct_telnet_timetrap_SUITE.erl @@ -4,7 +4,7 @@ -include_lib("common_test/include/ct.hrl"). --define(name,erl_telnet_server). +-define(name, telnet_server_conn1). %%-------------------------------------------------------------------- %% TEST SERVER CALLBACK FUNCTIONS @@ -17,6 +17,8 @@ end_per_suite(_Config) -> ok. suite() -> [{require,?name,{unix,[telnet]}}, + {require,ct_conn_log}, + {ct_hooks, [{cth_conn_log,[]}]}, {timetrap,{seconds,7}}]. all() -> diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 8e4852369d..9882fa980c 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -65,12 +65,12 @@ groups() -> []. init_per_group(_GroupName, Config) -> - Config. + Config. end_per_group(_GroupName, Config) -> - Config. + Config. + - %%-------------------------------------------------------------------- %% TEST CASES @@ -104,7 +104,7 @@ ts_if_1(Config) when is_list(Config) -> TestEvents = events_to_check(ts_if_1), ok = ct_test_support:verify_events(TestEvents, Events, Config). - + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -112,17 +112,17 @@ ts_if_1(Config) when is_list(Config) -> setup(Test, Config) -> Opts0 = ct_test_support:get_opts(Config), -% Level = ?config(trace_level, Config), -% EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], -% Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}], + % Level = ?config(trace_level, Config), + % EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + % Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}], Opts = [Test | Opts0], ERPid = ct_test_support:start_event_receiver(Config), {Opts,ERPid}. reformat(Events, EH) -> ct_test_support:reformat(Events, EH). -%reformat(Events, _EH) -> -% Events. + %reformat(Events, _EH) -> + % Events. %%%----------------------------------------------------------------- %%% TEST EVENTS @@ -140,14 +140,15 @@ test_events(ts_if_1) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{10,6,26}}, + {?eh,start_info,{10,8,25}}, + {?eh,tc_start,{ts_if_1_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ts_if_1_SUITE,tc1}}, - {?eh,tc_done,{ts_if_1_SUITE,tc1,{skipped, - {failed, - {ts_if_1_SUITE,init_per_testcase, - {timetrap_timeout,2000}}}}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc1, + {auto_skipped, + {failed, + {ts_if_1_SUITE,init_per_testcase,{timetrap_timeout,2000}}}}}}, {?eh,test_stats,{0,0,{0,1}}}, {?eh,tc_start,{ts_if_1_SUITE,tc2}}, {?eh,tc_done,{ts_if_1_SUITE,tc2, @@ -159,115 +160,162 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, {?eh,test_stats,{1,2,{0,1}}}, - {?eh,tc_done,{ts_if_1_SUITE,tc5,{skipped,{sequence_failed,seq1,tc4}}}}, - {?eh,test_stats,{1,2,{1,1}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}}, + {?eh,test_stats,{1,2,{0,2}}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]}}}, {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,seq2,[sequence]},ok}}, {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{1,3,{1,1}}}, + {?eh,test_stats,{1,3,{0,2}}}, {?eh,tc_auto_skip,{ts_if_1_SUITE,tc5,{failed,{ts_if_1_SUITE,tc4}}}}, - {?eh,test_stats,{1,3,{1,2}}}, + {?eh,test_stats,{1,3,{0,3}}}, {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,seq2,[sequence]},ok}}], {?eh,tc_start,{ts_if_1_SUITE,tc6}}, - {?eh,tc_done,{ts_if_1_SUITE,tc6,{skipped,{require_failed,{not_available,void}}}}}, - {?eh,test_stats,{1,3,{1,3}}}, - {?eh,tc_start,{ts_if_1_SUITE,tc7}}, - {?eh,tc_done,{ts_if_1_SUITE,tc7,ok}}, - {?eh,test_stats,{2,3,{1,3}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc6,{auto_skipped,{require_failed, + {not_available,void}}}}}, + {?eh,test_stats,{1,3,{0,4}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc7,{auto_skipped, + {testcase0_failed,bad_return_value}}}}, + {?eh,test_stats,{1,3,{0,5}}}, {?eh,tc_start,{ts_if_1_SUITE,tc8}}, {?eh,tc_done,{ts_if_1_SUITE,tc8,{skipped,"tc8 skipped"}}}, - {?eh,test_stats,{2,3,{2,3}}}, + {?eh,test_stats,{1,3,{1,5}}}, {?eh,tc_start,{ts_if_1_SUITE,tc9}}, {?eh,tc_done,{ts_if_1_SUITE,tc9,{skipped,'tc9 skipped'}}}, - {?eh,test_stats,{2,3,{3,3}}}, + {?eh,test_stats,{1,3,{2,5}}}, {?eh,tc_start,{ts_if_1_SUITE,tc10}}, - {?eh,tc_done,{ts_if_1_SUITE,tc10,{failed,{error,{function_clause,'_'}}}}}, - {?eh,test_stats,{2,4,{3,3}}}, + {?eh,tc_done,{ts_if_1_SUITE,tc10, + {failed,{error,{function_clause,'_'}}}}}, + {?eh,test_stats,{1,4,{2,5}}}, {?eh,tc_start,{ts_if_1_SUITE,tc11}}, {?eh,tc_done,{ts_if_1_SUITE,tc11, - {skipped,{failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}}, - {?eh,test_stats,{2,4,{3,4}}}, + {auto_skipped, + {failed,{ts_if_1_SUITE,init_per_testcase,bad_return}}}}}, + {?eh,test_stats,{1,4,{2,6}}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]},{skipped,g1_got_skipped}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, - {?eh,test_stats,{2,4,{3,5}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g1_got_skipped}}], - + {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g1,[]}, + {skipped,g1_got_skipped}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,gtc1,g1_got_skipped}}, + {?eh,test_stats,{1,4,{3,6}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g1},g1_got_skipped}}], + {parallel, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g2,[parallel]},ok}}, [{?eh,tc_start,{ts_if_1_SUITE,{init_per_group,g3,[]}}}, - {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, - {?eh,test_stats,{2,4,{3,6}}}, - {?eh,tc_auto_skip,{ts_if_1_SUITE,end_per_group,g3_got_skipped}}], + {?eh,tc_done,{ts_if_1_SUITE,{init_per_group,g3,[]},{skipped,g3_got_skipped}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,gtc2,g3_got_skipped}}, + {?eh,test_stats,{1,4,{4,6}}}, + {?eh,tc_user_skip,{ts_if_1_SUITE,{end_per_group,g3},g3_got_skipped}}], {?eh,tc_start,{ts_if_1_SUITE,{end_per_group,g2,[parallel]}}}, {?eh,tc_done,{ts_if_1_SUITE,{end_per_group,g2,[parallel]},ok}}]}, {?eh,tc_start,{ts_if_1_SUITE,tc12}}, {?eh,tc_done,{ts_if_1_SUITE,tc12,{failed,{testcase_aborted,'stopping tc12'}}}}, - {?eh,test_stats,{2,5,{3,6}}}, + {?eh,test_stats,{1,5,{4,6}}}, {?eh,tc_start,{ts_if_1_SUITE,tc13}}, {?eh,tc_done,{ts_if_1_SUITE,tc13,ok}}, - {?eh,test_stats,{3,5,{3,6}}}, + {?eh,test_stats,{2,5,{4,6}}}, {?eh,tc_start,{ts_if_1_SUITE,end_per_suite}}, {?eh,tc_done,{ts_if_1_SUITE,end_per_suite,ok}}, + {?eh,tc_start,{ts_if_2_SUITE,init_per_suite}}, - {?eh,tc_done,{ts_if_2_SUITE,init_per_suite, - {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, - {?eh,tc_auto_skip,{ts_if_2_SUITE,my_test_case, - {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, - {?eh,test_stats,{3,5,{3,7}}}, - {?eh,tc_auto_skip,{ts_if_2_SUITE,end_per_suite, - {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, + {?eh,tc_done, + {ts_if_2_SUITE,init_per_suite, + {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, + {?eh,tc_auto_skip, + {ts_if_2_SUITE,my_test_case, + {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, + {?eh,test_stats,{2,5,{4,7}}}, + {?eh,tc_auto_skip, + {ts_if_2_SUITE,end_per_suite, + {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,5,{4,7}}}, + {?eh,test_stats,{2,6,{4,7}}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{3,5,{5,7}}}, + {?eh,test_stats,{2,7,{4,7}}}, + {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_5_SUITE,init_per_suite, - {skipped,{require_failed_in_suite0,{not_available,undef_variable}}}}}, - {?eh,tc_auto_skip,{ts_if_5_SUITE,my_test_case, - {require_failed_in_suite0,{not_available,undef_variable}}}}, - {?eh,test_stats,{3,5,{5,8}}}, - {?eh,tc_auto_skip,{ts_if_5_SUITE,end_per_suite, - {require_failed_in_suite0,{not_available,undef_variable}}}}, + {auto_skipped, + {require_failed_in_suite0,{not_available,undef_variable}}}}}, + {?eh,tc_auto_skip, + {ts_if_5_SUITE,my_test_case, + {require_failed_in_suite0,{not_available,undef_variable}}}}, + {?eh,test_stats,{2,7,{4,8}}}, + {?eh,tc_auto_skip, + {ts_if_5_SUITE,end_per_suite, + {require_failed_in_suite0,{not_available,undef_variable}}}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, {?eh,tc_done,{ct_framework,init_per_suite, {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,tc_auto_skip,{ts_if_6_SUITE,tc1, - {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,test_stats,{3,5,{5,9}}}, + {?eh,tc_auto_skip, + {ts_if_6_SUITE,tc1, + {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, + {?eh,test_stats,{2,7,{4,9}}}, + {?eh,tc_auto_skip, + {ct_framework,end_per_suite, + {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,tc_start,{ts_if_7_SUITE,tc1}}, - {?eh,tc_done,{ts_if_7_SUITE,tc1,ok}}, - {?eh,test_stats,{4,5,{5,9}}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_done, + {ts_if_7_SUITE,tc1,{auto_skipped,{testcase0_failed,bad_return_value}}}}, + {?eh,test_stats,{2,7,{4,10}}}, + {?eh,tc_done,{ts_if_7_SUITE, + {init_per_group,g1,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,tc_auto_skip, + {ts_if_7_SUITE,tc2,{group0_failed,bad_return_value}}}, + {?eh,test_stats,{2,7,{4,11}}}, + {?eh,tc_auto_skip, + {ts_if_7_SUITE,{end_per_group,g1},{group0_failed,bad_return_value}}}, + + [{?eh,tc_start,{ts_if_7_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_done,{ts_if_7_SUITE,tc2, + {auto_skipped,{testcase0_failed,bad_return_value}}}}, + {?eh,test_stats,{2,7,{4,12}}}, + {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}], + + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + + + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_start,{ts_if_8_SUITE,tc1}}, {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{4,6,{5,9}}}, - + {?eh,test_stats,{2,8,{4,12}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + + {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, - {?eh,test_stats,{4,6,{6,9}}}, - + {?eh,test_stats,{2,8,{5,12}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, - {?eh,test_stats,{4,6,{7,9}}}, + {?eh,test_stats,{2,8,{6,12}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]. + diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl index a2254848d0..20e04c464b 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE_data/test_server_if/test/ts_if_7_SUITE.erl @@ -73,7 +73,8 @@ end_per_testcase(_TestCase, _Config) -> %% N = integer() | forever %%-------------------------------------------------------------------- groups() -> - []. + [{g1,[],[tc2]}, + {g2,[],[tc2]}]. %%-------------------------------------------------------------------- %% Function: all() -> GroupsAndTestCases | {skip,Reason} @@ -83,7 +84,12 @@ groups() -> %% Reason = term() %%-------------------------------------------------------------------- all() -> - [tc1]. + [tc1,{group,g1},{group,g2}]. + +group(g1) -> + exit(g1_byebye); +group(_) -> + []. tc1() -> exit(tc1_byebye). @@ -91,3 +97,9 @@ tc1() -> tc1(_) -> done. +tc2() -> + exit(tc2_byebye). + +tc2(_) -> + done. + diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 67e430f821..2e2b45d59f 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -40,6 +40,8 @@ -export([ct_test_halt/1, ct_rpc/2]). +-export([random_error/1]). + -include_lib("kernel/include/file.hrl"). %%%----------------------------------------------------------------- @@ -110,7 +112,6 @@ start_slave(NodeName, Config, Level) -> _ -> ok end, - TraceFile = filename:join(DataDir, "ct.trace"), case file:read_file_info(TraceFile) of {ok,_} -> @@ -395,6 +396,55 @@ ct_rpc({M,F,A}, Config) -> %%%----------------------------------------------------------------- +%%% random_error/1 +random_error(Config) when is_list(Config) -> + random:seed(now()), + Gen = fun(0,_) -> ok; (N,Fun) -> Fun(N-1, Fun) end, + Gen(random:uniform(100), Gen), + + ErrorTypes = ['BADMATCH','BADARG','CASE_CLAUSE','FUNCTION_CLAUSE', + 'EXIT','THROW','UNDEF'], + Type = lists:nth(random:uniform(length(ErrorTypes)), ErrorTypes), + Where = case random:uniform(2) of + 1 -> + io:format("ct_test_support *returning* error of type ~w", + [Type]), + tc; + 2 -> + io:format("ct_test_support *generating* error of type ~w", + [Type]), + lib + end, + ErrorFun = + fun() -> + case Type of + 'BADMATCH' -> + ok = proplists:get_value(undefined, Config); + 'BADARG' -> + size(proplists:get_value(priv_dir, Config)); + 'FUNCTION_CLAUSE' -> + random_error(x); + 'EXIT' -> + spawn_link(fun() -> + undef_proc ! hello, + ok + end); + 'THROW' -> + PrivDir = proplists:get_value(priv_dir, Config), + if is_list(PrivDir) -> throw(generated_throw) end; + 'UNDEF' -> + apply(?MODULE, random_error, []) + end + end, + %% either call the fun here or return it to the caller (to be + %% executed in a test case instead) + case Where of + tc -> ErrorFun; + lib -> ErrorFun() + end. + + +%%%----------------------------------------------------------------- %%% EVENT HANDLING handle_event(EH, Event) -> @@ -692,8 +742,10 @@ locate({parallel,TEvs}, Node, Evs, Config) -> test_server:format("Found ~p!", [TEv]), {Done,RemEvs2,length(RemEvs2)} end; - %% end_per_group auto skipped - (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) -> + %% end_per_group auto- or user skipped + (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + when AutoOrUserSkip == tc_auto_skip; + AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, @@ -704,10 +756,18 @@ locate({parallel,TEvs}, Node, Evs, Config) -> match -> false; _ -> true end; + ({EH,#event{name=tc_user_skip, + node=EvNode, + data={Mod,end_per_group,Reason}}}) when + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> - exit({tc_auto_skip_not_found,TEv}); + exit({tc_auto_or_user_skip_not_found,TEv}); (_) -> true end, RemEvs), @@ -925,8 +985,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> test_server:format("Found ~p!", [TEv]), {Done,RemEvs2,length(RemEvs2)} end; - %% end_per_group auto skipped - (TEv={TEH,tc_auto_skip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) -> + %% end_per_group auto-or user skipped + (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + when AutoOrUserSkip == tc_auto_skip; + AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, @@ -934,6 +996,11 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> data={Mod,end_per_group,Reason}}}) when EH == TEH, EvNode == Node, Mod == M, Reason == R -> false; + ({EH,#event{name=tc_user_skip, + node=EvNode, + data={Mod,end_per_group,Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, Reason == R -> + false; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -1178,6 +1245,9 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev, log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); +log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> + io:format(Dev, "~s~p],~n", [Ind,E]), + log_events1(Evs, Dev, Ind--" "); log_events1([E], Dev, Ind) -> io:format(Dev, "~s~p~n].~n", [Ind,E]), ok; diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index 6a4a4acd80..187b5e6d3a 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -760,16 +760,42 @@ test_events(all_groups) -> test_events(skip_all_groups) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,12}}, {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, {?eh,test_stats,{0,0,{1,0}}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, {?eh,test_stats,{0,0,{2,0}}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, {?eh,test_stats,{0,0,{3,0}}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, {?eh,test_stats,{0,0,{4,0}}}, - {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{5,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{6,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{7,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{8,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{9,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{10,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{11,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{12,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -788,21 +814,35 @@ test_events(group) -> test_events(skip_group) -> [ - {?eh,start_logging,'_'}, - {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - - {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, - {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, - {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, - {?eh,test_stats,{2,0,{0,0}}}, - {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}, - - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}}, - %%! But not test_group_7 since it's a sub-group! - {?eh,test_stats,{2,0,{2,0}}}, - {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}} - ]; + {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,8}}, + {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, + [{?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}}, + {?eh,tc_done,{groups_11_SUITE,{init_per_group,test_group_1a,[]},ok}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1a}}, + {?eh,tc_start,{groups_11_SUITE,testcase_1b}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{groups_11_SUITE,{end_per_group,test_group_1a,[]}}}, + {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},ok}}], + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{2,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_2}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_3b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_2b,"SKIPPED!"}}, + {?eh,test_stats,{2,0,{6,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_2}, + "SKIPPED!"}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}}, + {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} + ]; test_events(group_all_testcases) -> [ @@ -820,11 +860,23 @@ test_events(group_all_testcases) -> test_events(skip_group_all_testcases) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,4}}, {?eh,tc_start,{groups_11_SUITE,init_per_suite}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}}, - {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1a}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, {?eh,test_stats,{0,0,{2,0}}}, - {?eh,tc_start,{groups_11_SUITE,end_per_suite}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1a}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,{init_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{4,0}}}, + {?eh,tc_user_skip,{groups_11_SUITE,{end_per_group,test_group_1b}, + "SKIPPED!"}}, + {?eh,tc_done,{groups_11_SUITE,end_per_suite,init}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -934,7 +986,7 @@ test_events(subgroup) -> [ {?eh,start_logging,'_'}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - + {parallel, [{?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}}, @@ -962,36 +1014,58 @@ test_events(subgroup) -> test_events(skip_subgroup) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,6}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {parallel, - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}}, - {parallel, - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}}, - [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}], - {?eh,tc_user_skip, - {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}}, - {?eh,tc_start, - {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}} - ]}, - {?eh,tc_start, - {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}}, - {?eh,tc_done, - {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}], + [{?eh,tc_start,{groups_12_SUITE, + {init_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {init_per_group,test_group_5,[parallel]},ok}}, + + {parallel, + [{?eh,tc_start,{groups_12_SUITE, + {init_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {init_per_group,test_group_6,[parallel]},ok}}, + + [{?eh,tc_start,{groups_12_SUITE, + {init_per_group,test_group_7,[sequence]}}}, + {?eh,tc_done,{groups_12_SUITE, + {init_per_group,test_group_7,[sequence]},ok}}, + {?eh,tc_done,{groups_12_SUITE,testcase_7a,ok}}, + {?eh,tc_done,{groups_12_SUITE,testcase_7b,ok}}, + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_7,[sequence]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_7,[sequence]},ok}}], + + {?eh,tc_user_skip,{groups_12_SUITE, + {init_per_group,test_group_8},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_8},"SKIPPED!"}}, + + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_6,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_6,[parallel]},ok}}]}, + + {?eh,test_stats,{4,0,{2,0}}}, + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_5,[parallel]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_5,[parallel]},ok}}]}, - {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, + {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}], + + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, + {?eh,tc_done,{groups_12_SUITE,end_per_suite,init}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -1066,17 +1140,26 @@ test_events(subgroup_all_testcases) -> test_events(skip_subgroup_all_testcases) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,6}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - - [{?eh,tc_start, - {groups_12_SUITE,{init_per_group,test_group_4,[]}}}, - {?eh,tc_done, - {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, - {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}}, - {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}, - {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}} - ], - + [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE,{init_per_group,test_group_4,[]},ok}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {init_per_group,test_group_5},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{6,0}}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_5},"SKIPPED!"}}, + {?eh,tc_start,{groups_12_SUITE, + {end_per_group,test_group_4,[]}}}, + {?eh,tc_done,{groups_12_SUITE, + {end_per_group,test_group_4,[]},ok}}], + {?eh,tc_start,{groups_12_SUITE,end_per_suite}}, {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; @@ -1194,13 +1277,30 @@ test_events(skip_subgroup_testcase) -> test_events(sub_skipped_by_top) -> [ {?eh,start_logging,'_'}, + {?eh,start_info,{1,1,12}}, {?eh,tc_start,{groups_12_SUITE,init_per_suite}}, - - {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, - {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}}, - + {?eh,tc_user_skip,{groups_12_SUITE,{init_per_group,test_group_4}, + "SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {init_per_group,test_group_4},"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_7b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8a,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_8b,"SKIPPED!"}}, + {?eh,tc_user_skip,{groups_12_SUITE,testcase_5b,"SKIPPED!"}}, + {?eh,test_stats,{0,0,{12,0}}}, + {?eh,tc_user_skip,{groups_12_SUITE, + {end_per_group,test_group_4},"SKIPPED!"}}, {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}, - {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}} ]; diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl index 5fa187e5b4..9f8ca84e45 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE.erl @@ -1539,7 +1539,7 @@ flat_spec1_events() -> {?eh,test_stats,{1,2,{0,0}}}, {?eh,tc_start,{t11_SUITE,autoskip_tc}}, {?eh,tc_done, - {t11_SUITE,autoskip_tc,{skipped, + {t11_SUITE,autoskip_tc,{auto_skipped, {failed, {t11_SUITE,init_per_testcase, {kaboom,'_'}}}}}}, @@ -1562,7 +1562,7 @@ flat_spec1_events() -> {?eh,test_stats,{2,4,{1,1}}}, {?eh,tc_start,{t21_SUITE,autoskip_tc}}, {?eh,tc_done, - {t21_SUITE,autoskip_tc,{skipped, + {t21_SUITE,autoskip_tc,{auto_skipped, {failed, {t21_SUITE,init_per_testcase, {kaboom,'_'}}}}}}, @@ -1589,7 +1589,7 @@ flat_spec2_events() -> {?eh,test_stats,{1,2,{0,0}}}, {?eh,tc_start,{t12_SUITE,autoskip_tc}}, {?eh,tc_done, - {t12_SUITE,autoskip_tc,{skipped, + {t12_SUITE,autoskip_tc,{auto_skipped, {failed, {t12_SUITE,init_per_testcase, {kaboom,'_'}}}}}}, @@ -1612,7 +1612,7 @@ flat_spec2_events() -> {?eh,test_stats,{2,4,{1,1}}}, {?eh,tc_start,{t12_SUITE,autoskip_tc}}, {?eh,tc_done, - {t12_SUITE,autoskip_tc,{skipped, + {t12_SUITE,autoskip_tc,{auto_skipped, {failed, {t12_SUITE,init_per_testcase, {kaboom,'_'}}}}}}, @@ -1635,7 +1635,7 @@ flat_spec2_events() -> {?eh,test_stats,{3,6,{2,2}}}, {?eh,tc_start,{t22_SUITE,autoskip_tc}}, {?eh,tc_done, - {t22_SUITE,autoskip_tc,{skipped, + {t22_SUITE,autoskip_tc,{auto_skipped, {failed, {t22_SUITE,init_per_testcase, {kaboom,'_'}}}}}}, diff --git a/lib/common_test/test/telnet_server.erl b/lib/common_test/test/telnet_server.erl index 31884aa182..1760100d8e 100644 --- a/lib/common_test/test/telnet_server.erl +++ b/lib/common_test/test/telnet_server.erl @@ -64,18 +64,19 @@ accept(#state{listen=LSock}=State) -> {Acceptor,Sock} when is_port(Sock) -> case init_client(State#state{client=Sock}) of stopped -> - io:format("telnet_server stopped\n"), + io:format("[telnet_server] telnet_server stopped\n"), ok; R -> - io:format("connection to client closed with reason ~p~n",[R]), + io:format("[telnet_server] connection to client" + "closed with reason ~p~n",[R]), accept(State) end; {Acceptor,closed} -> - io:format("listen socket closed unexpectedly, " + io:format("[telnet_server] listen socket closed unexpectedly, " "terminating telnet_server\n"), ok; stop -> - io:format("telnet_server stopped\n"), + io:format("[telnet_server] telnet_server stopped\n"), ok end. @@ -188,7 +189,7 @@ do_handle_data(_Data,State) -> check_user(User,State) -> case lists:keyfind(User,1,State#state.users) of {User,Pwd} -> - dbg("user ok\n"), + dbg("user ok\n"), send("Password: ",State), {ok,State#state{authorized={user,Pwd}}}; false -> @@ -223,6 +224,6 @@ get_line([],_) -> false. dbg(_F) -> - io:format(_F). + dbg(_F,[]). dbg(_F,_A) -> - io:format(_F,_A). + io:format("[telnet_server] "++_F,_A). diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index c810a0d98d..568405b110 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.7.3 +COMMON_TEST_VSN = 1.7.4 |