diff options
Diffstat (limited to 'lib/common_test')
96 files changed, 5353 insertions, 2215 deletions
diff --git a/lib/common_test/doc/specs/.gitignore b/lib/common_test/doc/specs/.gitignore new file mode 100644 index 0000000000..322eebcb06 --- /dev/null +++ b/lib/common_test/doc/specs/.gitignore @@ -0,0 +1 @@ +specs_*.xml diff --git a/lib/common_test/doc/src/Makefile b/lib/common_test/doc/src/Makefile index e495f587a3..e3e478ab7f 100644 --- a/lib/common_test/doc/src/Makefile +++ b/lib/common_test/doc/src/Makefile @@ -53,7 +53,8 @@ XML_REF3_FILES = ct.xml \ ct_slave.xml \ ct_property_test.xml \ ct_netconfc.xml \ - ct_hooks.xml + ct_hooks.xml \ + ct_testspec.xml XML_REF6_FILES = common_test_app.xml XML_PART_FILES = part.xml @@ -105,11 +106,17 @@ HTML_REF_MAN_FILE = $(HTMLDIR)/index.html TOP_PDF_FILE = $(PDFDIR)/$(APPLICATION)-$(VSN).pdf +SPECS_FILES = $(XML_REF3_FILES:%.xml=$(SPECDIR)/specs_%.xml) + +TOP_SPECS_FILE = specs.xml + # ---------------------------------------------------- # FLAGS # ---------------------------------------------------- XML_FLAGS += DVIPS_FLAGS += +SPECS_FLAGS = -I../../include -I../../../snmp/include \ + -I../../../kernel/include # ---------------------------------------------------- # Targets @@ -118,7 +125,7 @@ DVIPS_FLAGS += $(HTMLDIR)/%.gif: %.gif $(INSTALL_DATA) $< $@ -docs: pdf html man +docs: man pdf html $(TOP_PDF_FILE): $(XML_FILES) @@ -139,6 +146,7 @@ clean clean_docs: rm -f $(MAN3DIR)/* rm -f $(MAN6DIR)/* rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) + rm -f $(SPECDIR)/* rm -f errs core *~ # ---------------------------------------------------- diff --git a/lib/common_test/doc/src/basics_chapter.xml b/lib/common_test/doc/src/basics_chapter.xml index b349d93813..9be71ae5df 100644 --- a/lib/common_test/doc/src/basics_chapter.xml +++ b/lib/common_test/doc/src/basics_chapter.xml @@ -75,7 +75,7 @@ <p><c>Common Test</c> is also a very useful tool for white-box testing Erlang code (for example, module testing), as the test programs can call exported Erlang - functions directly. there is very little overhead required for + functions directly. There is very little overhead required for implementing basic test suites and executing simple tests. For black-box testing Erlang software, Erlang RPC and standard O&M interfaces can be used for example. diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 48ffe653e4..a3b3f927eb 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2003</year><year>2016</year> + <year>2003</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -224,7 +224,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:end_per_suite-1"><c>end_per_suite/1</c></seealso> + must also be defined.</p> <p>This configuration function is called as the first function in the suite. It typically contains initializations that are common for @@ -256,7 +258,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:init_per_suite-1"><c>init_per_suite/1</c></seealso> + must also be defined.</p> <p>This function is called as the last test case in the suite. It is meant to be used for cleaning up after @@ -360,7 +364,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:end_per_group-2"><c>end_per_group/2</c></seealso> + must also be defined.</p> <p>This configuration function is called before execution of a test case group. It typically contains initializations that are @@ -396,7 +402,9 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, then <seealso + marker="#Module:init_per_group-2"><c>init_per_group/2</c></seealso> + must also be defined.</p> <p>This function is called after the execution of a test case group is finished. It is meant to be used for cleaning up after @@ -427,7 +435,10 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, + then <seealso marker="#Module:end_per_testcase-2"> + <c>end_per_testcase/2</c></seealso> must also be + defined.</p> <p>This function is called before each test case. Argument <c>TestCase</c> is the test case name, and @@ -454,7 +465,10 @@ </type> <desc> - <p>OPTIONAL</p> + <p>OPTIONAL; if this function is defined, + then <seealso marker="#Module:init_per_testcase-2"> + <c>init_per_testcase/2</c></seealso> must also be + defined.</p> <p>This function is called after each test case, and can be used to clean up after diff --git a/lib/common_test/doc/src/ct.xml b/lib/common_test/doc/src/ct.xml index 53ef41dd5b..1a3cfdb0c5 100644 --- a/lib/common_test/doc/src/ct.xml +++ b/lib/common_test/doc/src/ct.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2010</year><year>2016</year> + <year>2010</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -70,17 +70,69 @@ <marker id="types"/> <taglist> - <tag><c>handle() = pid()</c></tag> - <item><marker id="type-handle"/> - <p>The identity (handle) of a connection.</p></item> - - <tag><c>target_name() = atom()</c></tag> - <item><marker id="type-target_name"/> - <p>A name and association to configuration data introduced - through a require statement, or a call to - <seealso marker="#require-2"><c>ct:require/2</c></seealso>, - for example, - <c>ct:require(mynodename,{node,[telnet]})</c>.</p></item> + <tag> + <marker id="type-handle"/> + <c>handle() = pid()</c> + </tag> + <item> + <p>The identity (handle) of a connection.</p> + </item> + + <tag> + <marker id="type-config_key"/> + <c>config_key() = atom()</c> + </tag> + <item> + <p>A configuration key which exists in a configuration file</p> + </item> + + <tag> + <marker id="type-target_name"/><c>target_name() = atom()</c> + </tag> + <item> + <p>A name and association to configuration data introduced + through a require statement, or a call to + <seealso marker="#require-2"><c>ct:require/2</c></seealso>, + for example, + <c>ct:require(mynodename,{node,[telnet]})</c>.</p> + </item> + + <tag> + <marker id="type-key_or_name"/> + <c>key_or_name() = config_key() | target_name()</c> + </tag> + <item/> + + <tag> + <marker id="type-conn_log_options"/> + <c>conn_log_options() = [conn_log_option()]</c> + </tag> + <item> + <p>Options that can be given to the <c>cth_conn_log</c> hook, + which is used for logging of NETCONF and Telnet + connections. See + <seealso marker="ct_netconfc#Logging">ct_netconfc</seealso> + or <seealso marker="ct_telnet#Logging">ct_telnet</seealso> + for description and examples of how to use this hook.</p> + </item> + + <tag> + <marker id="type-conn_log_option"/> + <c>conn_log_option() = {log_type,conn_log_type()} | {hosts,[key_or_name()]}</c> + </tag> + <item/> + + <tag> + <marker id="type-conn_log_type"/> + <c>conn_log_type() = raw | pretty | html | silent</c> + </tag> + <item/> + + <tag> + <marker id="type-conn_log_mod"/> + <c>conn_log_mod() = ct_netconfc | ct_telnet</c> + </tag> + <item/> </taglist> </section> @@ -740,7 +792,7 @@ <v>Format = string()</v> <v>FormatArgs = list()</v> <v>Opts = [Opt]</v> - <v>Opt = no_css | esc_chars</v> + <v>Opt = {heading,string()} | no_css | esc_chars</v> </type> <desc><marker id="log-5"/> <p>Prints from a test case to the log file.</p> @@ -798,53 +850,71 @@ <func> <name>pal(Format) -> ok</name> - <fsummary>Equivalent to pal(default, 50, Format, []).</fsummary> + <fsummary>Equivalent to pal(default, 50, Format, [], []).</fsummary> <desc><marker id="pal-1"/> <p>Equivalent to - <seealso marker="#pal-4"><c>ct:pal(default, 50, Format, - [])</c></seealso>.</p> + <seealso marker="#pal-5"><c>ct:pal(default, 50, Format, + [], [])</c></seealso>.</p> </desc> </func> <func> <name>pal(X1, X2) -> ok</name> <fsummary>Equivalent to pal(Category, Importance, Format, - FormatArgs).</fsummary> + FormatArgs, []).</fsummary> <type> <v>X1 = Category | Importance | Format</v> <v>X2 = Format | FormatArgs</v> </type> <desc><marker id="pal-2"/> - <p>Equivalent to <seealso marker="#pal-4"><c>ct:pal(Category, - Importance, Format, FormatArgs)</c></seealso>.</p> + <p>Equivalent to <seealso marker="#pal-5"><c>ct:pal(Category, + Importance, Format, FormatArgs, [])</c></seealso>.</p> </desc> </func> <func> <name>pal(X1, X2, X3) -> ok</name> <fsummary>Equivalent to pal(Category, Importance, Format, - FormatArgs).</fsummary> + FormatArgs, Opts).</fsummary> <type> <v>X1 = Category | Importance</v> <v>X2 = Importance | Format</v> - <v>X3 = Format | FormatArgs</v> + <v>X3 = Format | FormatArgs | Opts</v> </type> <desc><marker id="pal-3"/> - <p>Equivalent to <seealso marker="#pal-4"><c>ct:pal(Category, - Importance, Format, FormatArgs)</c></seealso>.</p> + <p>Equivalent to <seealso marker="#pal-5"><c>ct:pal(Category, + Importance, Format, FormatArgs, Opts)</c></seealso>.</p> </desc> </func> <func> - <name>pal(Category, Importance, Format, FormatArgs) -> ok</name> + <name>pal(X1, X2, X3, X4) -> ok</name> + <fsummary>Equivalent to pal(Category, Importance, Format, + FormatArgs, Opts).</fsummary> + <type> + <v>X1 = Category | Importance</v> + <v>X2 = Importance | Format</v> + <v>X3 = Format | FormatArgs</v> + <v>X4 = FormatArgs | Opts</v> + </type> + <desc><marker id="pal-4"/> + <p>Equivalent to <seealso marker="#pal-5"><c>ct:pal(Category, + Importance, Format, FormatArgs, Opts)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>pal(Category, Importance, Format, FormatArgs, Opts) -> ok</name> <fsummary>Prints and logs from a test case.</fsummary> <type> <v>Category = atom()</v> <v>Importance = integer()</v> <v>Format = string()</v> <v>FormatArgs = list()</v> + <v>Opts = [Opt]</v> + <v>Opt = {heading,string()} | no_css</v> </type> - <desc><marker id="pal-4"/> + <desc><marker id="pal-5"/> <p>Prints and logs from a test case.</p> <p>This function is meant for printing a string from a test case, @@ -888,52 +958,70 @@ <func> <name>print(Format) -> ok</name> - <fsummary>Equivalent to print(default, 50, Format, []).</fsummary> + <fsummary>Equivalent to print(default, 50, Format, [], []).</fsummary> <desc><marker id="print-1"/> - <p>Equivalent to <seealso marker="#print-4"><c>ct:print(default, - 50, Format, [])</c></seealso>.</p> + <p>Equivalent to <seealso marker="#print-5"><c>ct:print(default, + 50, Format, [], [])</c></seealso>.</p> </desc> </func> <func> <name>print(X1, X2) -> ok</name> <fsummary>Equivalent to print(Category, Importance, Format, - FormatArgs).</fsummary> + FormatArgs, []).</fsummary> <type> <v>X1 = Category | Importance | Format</v> <v>X2 = Format | FormatArgs</v> </type> <desc><marker id="print-2"/> - <p>Equivalent to <seealso marker="#print-4"><c>ct:print(Category, - Importance, Format, FormatArgs)</c></seealso>.</p> + <p>Equivalent to <seealso marker="#print-5"><c>ct:print(Category, + Importance, Format, FormatArgs, [])</c></seealso>.</p> </desc> </func> <func> <name>print(X1, X2, X3) -> ok</name> <fsummary>Equivalent to print(Category, Importance, Format, - FormatArgs).</fsummary> + FormatArgs, Opts).</fsummary> <type> <v>X1 = Category | Importance</v> <v>X2 = Importance | Format</v> - <v>X3 = Format | FormatArgs</v> + <v>X3 = Format | FormatArgs | Opts</v> </type> <desc><marker id="print-3"/> - <p>Equivalent to <seealso marker="#print-4"><c>ct:print(Category, - Importance, Format, FormatArgs)</c></seealso>.</p> + <p>Equivalent to <seealso marker="#print-5"><c>ct:print(Category, + Importance, Format, FormatArgs, Opts)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>print(X1, X2, X3, X4) -> ok</name> + <fsummary>Equivalent to print(Category, Importance, Format, + FormatArgs, Opts).</fsummary> + <type> + <v>X1 = Category | Importance</v> + <v>X2 = Importance | Format</v> + <v>X3 = Format | FormatArgs</v> + <v>X4 = FormatArgs | Opts</v> + </type> + <desc><marker id="print-4"/> + <p>Equivalent to <seealso marker="#print-5"><c>ct:print(Category, + Importance, Format, FormatArgs, Opts)</c></seealso>.</p> </desc> </func> <func> - <name>print(Category, Importance, Format, FormatArgs) -> ok</name> + <name>print(Category, Importance, Format, FormatArgs, Opts) -> ok</name> <fsummary>Prints from a test case to the console.</fsummary> <type> <v>Category = atom()</v> <v>Importance = integer()</v> <v>Format = string()</v> <v>FormatArgs = list()</v> + <v>Opts = [Opt]</v> + <v>Opt = {heading,string()}</v> </type> - <desc><marker id="print-4"/> + <desc><marker id="print-5"/> <p>Prints from a test case to the console.</p> <p>This function is meant for printing a string from a test case to @@ -1154,7 +1242,7 @@ Opts.</fsummary> <type> <v>Opts = [OptTuples]</v> - <v>OptTuples = {dir, TestDirs} | {suite, Suites} | {group, Groups} | {testcase, Cases} | {spec, TestSpecs} | {join_specs, Bool} | {label, Label} | {config, CfgFiles} | {userconfig, UserConfig} | {allow_user_terms, Bool} | {logdir, LogDir} | {silent_connections, Conns} | {stylesheet, CSSFile} | {cover, CoverSpecFile} | {cover_stop, Bool} | {step, StepOpts} | {event_handler, EventHandlers} | {include, InclDirs} | {auto_compile, Bool} | {abort_if_missing_suites, Bool} | {create_priv_dir, CreatePrivDir} | {multiply_timetraps, M} | {scale_timetraps, Bool} | {repeat, N} | {duration, DurTime} | {until, StopTime} | {force_stop, ForceStop} | {decrypt, DecryptKeyOrFile} | {refresh_logs, LogDir} | {logopts, LogOpts} | {verbosity, VLevels} | {basic_html, Bool} | {esc_chars, Bool} | {ct_hooks, CTHs} | {enable_builtin_hooks, Bool} | {release_shell, Bool}</v> + <v>OptTuples = {dir, TestDirs} | {suite, Suites} | {group, Groups} | {testcase, Cases} | {spec, TestSpecs} | {join_specs, Bool} | {label, Label} | {config, CfgFiles} | {userconfig, UserConfig} | {allow_user_terms, Bool} | {logdir, LogDir} | {silent_connections, Conns} | {stylesheet, CSSFile} | {cover, CoverSpecFile} | {cover_stop, Bool} | {step, StepOpts} | {event_handler, EventHandlers} | {include, InclDirs} | {auto_compile, Bool} | {abort_if_missing_suites, Bool} | {create_priv_dir, CreatePrivDir} | {multiply_timetraps, M} | {scale_timetraps, Bool} | {repeat, N} | {duration, DurTime} | {until, StopTime} | {force_stop, ForceStop} | {decrypt, DecryptKeyOrFile} | {refresh_logs, LogDir} | {logopts, LogOpts} | {verbosity, VLevels} | {basic_html, Bool} | {esc_chars, Bool} | {keep_logs,KeepSpec} | {ct_hooks, CTHs} | {enable_builtin_hooks, Bool} | {release_shell, Bool}</v> <v>TestDirs = [string()] | string()</v> <v>Suites = [string()] | [atom()] | string() | atom()</v> <v>Cases = [atom()] | atom()</v> @@ -1190,6 +1278,7 @@ <v>VLevels = VLevel | [{Category, VLevel}]</v> <v>VLevel = integer()</v> <v>Category = atom()</v> + <v>KeepSpec = all | pos_integer()</v> <v>CTHs = [CTHModule | {CTHModule, CTHInitArgs}]</v> <v>CTHModule = atom()</v> <v>CTHInitArgs = term()</v> diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index c2cf29c530..954be0ffba 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>2010</year><year>2016</year> + <year>2010</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -208,9 +208,10 @@ </func> <func> - <name>Module:pre_init_per_group(GroupName, InitData, CTHState) -> Result</name> + <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>InitData = Config | SkipOrFail</v> <v>Config = NewConfig = [{Key,Value}]</v> @@ -231,13 +232,19 @@ but for function <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso> instead.</p> + + <p>If <c>Module:pre_init_per_group/4</c> is not exported, common_test + will attempt to call <c>Module:pre_init_per_group(GroupName, + InitData, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -> Result</name> + <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -258,13 +265,19 @@ but for function <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso> instead.</p> + + <p>If <c>Module:post_init_per_group/5</c> is not exported, common_test + will attempt to call <c>Module:post_init_per_group(GroupName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -> Result</name> + <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -> Result</name> <fsummary>Called before init_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> <v>InitData = Config | SkipOrFail</v> <v>Config = NewConfig = [{Key,Value}]</v> @@ -286,6 +299,11 @@ <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso> instead.</p> + <p>If <c>Module:pre_init_per_testcase/4</c> is not exported, common_test + will attempt to call <c>Module:pre_init_per_testcase(TestcaseName, + InitData, CTHState)</c> instead. This is for backwards + compatibility.</p> + <p>CTHs cannot be added here right now. That feature may be added in a later release, but it would right now break backwards compatibility.</p> @@ -293,9 +311,10 @@ </func> <func> - <name>Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -> Result</name> + <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> <fsummary>Called after init_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -316,15 +335,21 @@ but for function <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso> instead.</p> + + <p>If <c>Module:post_init_per_testcase/5</c> is not exported, common_test + will attempt to call <c>Module:post_init_per_testcase(TestcaseName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -> Result</name> + <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> - <v>InitData = Config</v> + <v>EndData = Config</v> <v>Config = NewConfig = [{Key,Value}]</v> <v>CTHState = NewCTHState = term()</v> <v>Result = {NewConfig, NewCTHState}</v> @@ -345,14 +370,20 @@ <p>This function can not change the result of the test case by returning skip or fail tuples, but it may insert items in <c>Config</c> that can be read in - <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/4</c>.</p> + <c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p> + + <p>If <c>Module:pre_end_per_testcase/4</c> is not exported, common_test + will attempt to call <c>Module:pre_end_per_testcase(TestcaseName, + EndData, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -> Result</name> + <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_testcase.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestcaseName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -373,13 +404,19 @@ but for function <seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso> instead.</p> + + <p>If <c>Module:post_end_per_testcase/5</c> is not exported, common_test + will attempt to call <c>Module:post_end_per_testcase(TestcaseName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:pre_end_per_group(GroupName, EndData, CTHState) -> Result</name> + <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -> Result</name> <fsummary>Called before end_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>EndData = Config | SkipOrFail</v> <v>Config = NewConfig = [{Key,Value}]</v> @@ -400,13 +437,19 @@ but for function <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group</c></seealso> instead.</p> + + <p>If <c>Module:pre_end_per_group/4</c> is not exported, common_test + will attempt to call <c>Module:pre_end_per_group(GroupName, + EndData, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -> Result</name> + <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name> <fsummary>Called after end_per_group.</fsummary> <type> + <v>SuiteName = atom()</v> <v>GroupName = atom()</v> <v>Config = [{Key,Value}]</v> <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -427,6 +470,11 @@ but for function <seealso marker="common_test#Module:end_per_group-2">end_per_group</seealso> instead.</p> + + <p>If <c>Module:post_end_per_group/5</c> is not exported, common_test + will attempt to call <c>Module:post_end_per_group(GroupName, + Config, Return, CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> @@ -485,9 +533,10 @@ </func> <func> - <name>Module:on_tc_fail(TestName, Reason, CTHState) -> NewCTHState</name> + <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v> <v>FuncName = atom()</v> <v>GroupName = atom()</v> @@ -505,7 +554,7 @@ <item><p>If <c>init_per_suite</c> fails, this function is called after <seealso marker="#Module:post_init_per_suite-4"><c>post_init_per_suite</c></seealso>.</p></item> <item><p>If a test case fails, this funcion is called after - <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item> + <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item> </list> <p>If the failed test case belongs to a test case group, the first @@ -519,13 +568,19 @@ For details, see section <seealso marker="event_handler_chapter#events">Event Handling</seealso> in the User's Guide.</p> + + <p>If <c>Module:on_tc_fail/4</c> is not exported, common_test + will attempt to call <c>Module:on_tc_fail(TestName, Reason, + CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> <func> - <name>Module:on_tc_skip(TestName, Reason, CTHState) -> NewCTHState</name> + <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name> <fsummary>Called after the CTH scope ends.</fsummary> <type> + <v>SuiteName = atom()</v> <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v> <v>FuncName = atom()</v> <v>GroupName = atom()</v> @@ -542,9 +597,9 @@ <list type="bulleted"> <item><p>If <c>init_per_group</c> is skipped, this function is called after - <seealso marker="#Module:post_init_per_group-4"><c>post_init_per_group</c></seealso>.</p></item> + <seealso marker="#Module:post_init_per_group-5"><c>post_init_per_group</c></seealso>.</p></item> <item><p>If a test case is skipped, this function is called after - <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item> + <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item> </list> <p>If the skipped test case belongs to a test case group, the first @@ -559,6 +614,11 @@ For details, see section <seealso marker="event_handler_chapter#events">Event Handling</seealso> in the User's Guide.</p> + + <p>If <c>Module:on_tc_skip/4</c> is not exported, common_test + will attempt to call <c>Module:on_tc_skip(TestName, Reason, + CTHState)</c> instead. This is for backwards + compatibility.</p> </desc> </func> diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index 0e4c35e11f..7ecc2e4298 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2011</year><year>2016</year> + <year>2011</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -38,7 +38,7 @@ extensions of the default behavior of <c>Common Test</c> using hooks before and after all test suite calls. CTHs allow advanced <c>Common Test</c> users to abstract out behavior that is common to multiple test suites - without littering all test suites with library calls. this can be used + without littering all test suites with library calls. This can be used for logging, starting, and monitoring external systems, building C files needed by the tests, and so on.</p> @@ -175,10 +175,10 @@ <row> <cell><seealso marker="common_test#Module:init_per_group-2"> init_per_group/2</seealso></cell> - <cell><seealso marker="ct_hooks#Module:post_init_per_group-4"> - post_init_per_group/4</seealso> is called</cell> - <cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> - post_end_per_group/4</seealso> has been called for that group</cell> + <cell><seealso marker="ct_hooks#Module:post_init_per_group-5"> + post_init_per_group/5</seealso> is called</cell> + <cell><seealso marker="ct_hooks#Module:post_end_per_group-5"> + post_end_per_group/5</seealso> has been called for that group</cell> </row> <tcaption>Scope of a CTH</tcaption> </table> @@ -245,16 +245,18 @@ </list> <p> - This is done in the CTH functions called pre_<name of function>. - These functions take the same three arguments, <c>Name</c>, + This is done in the CTH functions called <c>pre_<name of function></c>. + These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable), <c>Config</c>, and <c>CTHState</c>. The return value of the CTH function is always a combination of a result for the suite/group/test and an updated <c>CTHState</c>.</p> <p>To let the test suite continue on executing, return the configuration - list that you want the test to use as the result. To skip or - fail the test, return a tuple with <c>skip</c> or <c>fail</c>, and a reason - as the result.</p> + list that you want the test to use as the result.</p> + + <p>All pre hooks, except <c>pre_end_per_testcase/4</c>, can + skip or fail the test by returning a tuple with <c>skip</c> or + <c>fail</c>, and a reason as the result.</p> <p><em>Example:</em></p> <code> @@ -290,7 +292,7 @@ <p> This is done in the CTH functions called <c>post_<name of function></c>. - These functions take the same four arguments, <c>Name</c>, + These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable), <c>Config</c>, <c>Return</c>, and <c>CTHState</c>. <c>Config</c> in this case is the same <c>Config</c> as the testcase is called with. <c>Return</c> is the value returned by the testcase. If the testcase @@ -308,7 +310,7 @@ <p><em>Example:</em></p> <code> - post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) -> case db:check_consistency() of true -> %% DB is good, pass the test. @@ -317,7 +319,7 @@ %% DB is not good, mark as skipped instead of failing {{skip, "DB is inconsisten!"}, CTHState} end; - post_end_per_testcase(_TC, Config, Return, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) -> %% Do nothing if tc does not crash. {Return, CTHState}.</code> @@ -331,8 +333,8 @@ <title>Skip and Fail Hooks</title> <p> After any post hook has been executed for all installed CTHs, - <seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso> - or <seealso marker="ct_hooks#Module:on_tc_skip-3">on_tc_skip</seealso> + <seealso marker="ct_hooks#Module:on_tc_fail-4">on_tc_fail</seealso> + or <seealso marker="ct_hooks#Module:on_tc_skip-4">on_tc_skip</seealso> is called if the testcase failed or was skipped, respectively. You cannot affect the outcome of the tests any further at this point. </p> @@ -389,18 +391,18 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). - -export([pre_init_per_group/3]). - -export([post_init_per_group/4]). - -export([pre_end_per_group/3]). - -export([post_end_per_group/4]). + -export([pre_init_per_group/4]). + -export([post_init_per_group/5]). + -export([pre_end_per_group/4]). + -export([post_end_per_group/5]). - -export([pre_init_per_testcase/3]). - -export([post_init_per_testcase/4]). - -export([pre_end_per_testcase/3]). - -export([post_end_per_testcase/4]). + -export([pre_init_per_testcase/4]). + -export([post_init_per_testcase/5]). + -export([pre_end_per_testcase/4]). + -export([post_end_per_testcase/5]). - -export([on_tc_fail/3]). - -export([on_tc_skip/3]). + -export([on_tc_fail/4]). + -export([on_tc_skip/4]). -export([terminate/1]). @@ -435,46 +437,46 @@ total = State#state.total + State#state.suite_total } }. %% @doc Called before each init_per_group. - pre_init_per_group(Group,Config,State) -> + pre_init_per_group(Suite,Group,Config,State) -> {Config, State}. %% @doc Called after each init_per_group. - post_init_per_group(Group,Config,Return,State) -> + post_init_per_group(Suite,Group,Config,Return,State) -> {Return, State}. %% @doc Called before each end_per_group. - pre_end_per_group(Group,Config,State) -> + pre_end_per_group(Suite,Group,Config,State) -> {Config, State}. %% @doc Called after each end_per_group. - post_end_per_group(Group,Config,Return,State) -> + post_end_per_group(Suite,Group,Config,Return,State) -> {Return, State}. %% @doc Called before each init_per_testcase. - pre_init_per_testcase(TC,Config,State) -> + pre_init_per_testcase(Suite,TC,Config,State) -> {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }. %% Called after each init_per_testcase (immediately before the test case). - post_init_per_testcase(TC,Config,Return,State) -> + post_init_per_testcase(Suite,TC,Config,Return,State) -> {Return, State} %% @doc Called before each end_per_testcase (immediately after the test case). - pre_end_per_testcase(TC,Config,State) -> + pre_end_per_testcase(Suite,TC,Config,State) -> {Config, State}. %% @doc Called after each end_per_testcase. - post_end_per_testcase(TC,Config,Return,State) -> - TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + post_end_per_testcase(Suite,TC,Config,Return,State) -> + TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)}, {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. - on_tc_fail(TC, Reason, State) -> + on_tc_fail(Suite, TC, Reason, State) -> State. %% @doc Called when a test case is skipped by either user action %% or due to an init function failing. - on_tc_skip(TC, Reason, State) -> + on_tc_skip(Suite, TC, Reason, State) -> State. %% @doc Called when the scope of the CTH is done diff --git a/lib/common_test/doc/src/ct_netconfc.xml b/lib/common_test/doc/src/ct_netconfc.xml index e6930b30d5..ab83f50733 100644 --- a/lib/common_test/doc/src/ct_netconfc.xml +++ b/lib/common_test/doc/src/ct_netconfc.xml @@ -35,13 +35,35 @@ <module>ct_netconfc</module> <modulesummary>NETCONF client module.</modulesummary> -<description> + <description> <p>NETCONF client module.</p> <p>The NETCONF client is compliant with RFC 4741 NETCONF Configuration Protocol and RFC 4742 Using the NETCONF Configuration Protocol over - Secure SHell (SSH)..</p> + Secure SHell (SSH).</p> + + <marker id="Connecting"/> + <p><em>Connecting to a NETCONF server</em></p> + + <p>NETCONF sessions can either be opened by a single call + to <seealso marker="#open-1"><c>open/1,2</c></seealso> or by a call + to <seealso marker="#connect-1"><c>connect/1,2</c></seealso> followed + by one or more calls to + <seealso marker="#session-1"><c>session/1,2,3</c></seealso>.</p> + + <p>The properties of the sessions will be exactly the same, except + that when + using <seealso marker="#connect-1"><c>connect/1,2</c></seealso>, you + may start multiple sessions over the same SSH connection. Each + session is implemented as an SSH channel.</p> + + <p><seealso marker="#open-1"><c>open/1,2</c></seealso> will establish one + SSH connection with one SSH channel implementing one NETCONF + session. You may start mutiple sessions by + calling <seealso marker="#open-1"><c>open/1,2</c></seealso> multiple + times, but then a new SSH connection will be established for each + session.</p> <p>For each server to test against, the following entry can be added to a configuration file:</p> @@ -49,23 +71,21 @@ <pre> {server_id(),options()}.</pre> - <p>The <c>server_id()</c> or an associated <c>target_name()</c> (see - module <seealso marker="ct"><c>ct</c></seealso>) must then be used - in calls to - <seealso marker="#open-2"><c>ct_netconfc:open/2</c></seealso>.</p> + <p>The <seealso marker="#type-server_id"><c>server_id()</c></seealso> + or an associated + <seealso marker="ct#type-target_name"><c>ct:target_name()</c></seealso> + must then be used in calls to + <seealso marker="#connect-2"><c>connect/2</c></seealso> + or <seealso marker="#open-2"><c>open/2</c></seealso>.</p> - <p>If no configuration exists for a server, a session can still be - opened by calling - <seealso marker="#open-2"><c>ct_netconfc:open/2</c></seealso> with - all necessary options specified in the call. The first argument to - <seealso marker="#open-2"><c>ct_netconfc:open/2</c></seealso> can - then be any atom.</p> + <p>If no configuration exists for a server, + use <seealso marker="#connect-1"><c>connect/1</c></seealso> + or <seealso marker="#open-1"><c>open/1</c></seealso> instead, + and specify all necessary options in the <c>Options</c> parameter.</p> - </description> + <marker id="Logging"/> + <p><em>Logging</em></p> - <section> - <marker id="Logging"/> - <title>Logging</title> <p>The NETCONF server uses <c>error_logger</c> for logging of NETCONF traffic. A special purpose error handler is implemented in <c>ct_conn_log_h</c>. To use this error handler, add the @@ -73,9 +93,9 @@ <pre> suite() -> - [{ct_hooks, [{cth_conn_log, [{conn_mod(),hook_options()}]}]}].</pre> + [{ct_hooks, [{cth_conn_log, [{<seealso marker="ct#type-conn_log_mod"><c>ct:conn_log_mod()</c></seealso>,<seealso marker="ct#type-conn_log_options"><c>ct:conn_log_options()</c></seealso>}]}]}].</pre> - <p><c>conn_mod()</c> is the name of the <c>Common Test</c> module + <p><c>conn_log_mod()</c> is the name of the <c>Common Test</c> module implementing the connection protocol, for example, <c>ct_netconfc</c>.</p> <p>Hook option <c>log_type</c> specifies the type of logging:</p> @@ -84,7 +104,7 @@ <tag><c>raw</c></tag> <item><p>The sent and received NETCONF data is logged to a separate text file "as is" without any formatting. A link to the file is - added to the test case HTML log.</p>.</item> + added to the test case HTML log.</p></item> <tag><c>pretty</c></tag> <item><p>The sent and received NETCONF data is logged to a separate @@ -104,7 +124,7 @@ To do this, use hook option <c>hosts</c> and list the names of the servers/connections to be used in the suite. The connections must be named for this to work, that is, they must be opened with - <seealso marker="#open-2"><c>ct_netconfc:open/2</c></seealso>.</p> + <seealso marker="#open-2"><c>open/2</c></seealso>.</p> <p>Option <c>hosts</c> has no effect if <c>log_type</c> is set to <c>html</c> or <c>silent</c>.</p> @@ -113,13 +133,13 @@ configuration variable <c>ct_conn_log</c>:</p> <pre> - {ct_conn_log,[{conn_mod(),hook_options()}]}.</pre> + {ct_conn_log,[{<seealso marker="ct#type-conn_log_mod"><c>ct:conn_log_mod()</c></seealso>,<seealso marker="ct#type-conn_log_options"><c>ct:conn_log_options()</c></seealso>}]}.</pre> <p>For example:</p> <pre> {ct_conn_log,[{ct_netconfc,[{log_type,pretty}, - {hosts,[key_or_name()]}]}]}</pre> + {hosts,[<seealso marker="ct#type-key_or_name"><c>ct:key_or_name()</c></seealso>]}]}]}</pre> <note> <p>Hook options specified in a configuration file overwrite the @@ -164,173 +184,149 @@ <p>The same <c>ct_hooks</c> statement without the configuration file would cause HTML logging of all NETCONF connections in to the test case HTML log.</p> - </section> - <section> - <marker id="Notifications"/> - <title>Notifications</title> + <marker id="Notifications"/> + <p><em>Notifications</em></p> <p>The NETCONF client is also compliant with RFC 5277 NETCONF Event Notifications, which defines a mechanism for an asynchronous message notification delivery service for the NETCONF protocol.</p> <p>Specific functions to support this are - <seealso marker="#create_subscription-6"><c>ct_netconfc:create_subscription/6</c></seealso> + <seealso marker="#create_subscription-1"><c>create_subscription/1-6</c></seealso> and - <seealso marker="#get_event_streams-3"><c>ct_netconfc:get_event_streams/3</c></seealso>. - (The functions also exist with other arities.)</p> - </section> - - <section> - <title>Data Types</title> - <marker id="types"/> - <taglist> - <tag><c>client() = handle() | key_or_name()</c></tag> - <item><marker id="type-client"/> - <p>For <c>handle()</c>, see module - <seealso marker="ct"><c>ct</c></seealso>.</p></item> - - <tag><c>error_reason() = term()</c></tag> - <item><marker id="type-error_reason"/> </item> - <tag><c>event_time() = {eventTime, xml_attributes(), [xs_datetime()]}</c></tag> - <item><marker id="type-event_time"/> </item> - - <tag><c>handle() = term()</c></tag> - <item><marker id="type-handle"/> - <p>Opaque reference for a connection (NETCONF session). For more - information, see module <seealso marker="ct"><c>ct</c></seealso>.</p> - </item> - - <tag><c>host() = </c><seealso marker="kernel:inet#type-hostname"><c>inet:hostname()</c></seealso> - <c> | </c><seealso marker="kernel:inet#type-ip_address"><c>inet:ip_address()</c></seealso></tag> - <item><marker id="type-host"/></item> - - <tag><c>key_or_name() = server_id() | target_name()</c></tag> - <item><marker id="type-key_or_name"/> - <p>For <c>target_name</c>, see module - <seealso marker="ct"><c>ct</c></seealso>.</p></item> - - <tag><c>netconf_db() = running | startup | candidate</c></tag> - <item><marker id="type-netconf_db"/> </item> + <seealso marker="#get_event_streams-1"><c>get_event_streams/1-3</c></seealso>.</p> - <tag><c>notification() = {notification, xml_attributes(), notification_content()}</c></tag> - <item><marker id="type-notification"/> </item> + <marker id="Default_timeout"/> + <p><em>Default Timeout</em></p> - <tag><c>notification_content() = [event_time() | simple_xml()]</c></tag> - <item><marker id="type-notification_content"/> </item> + <p>Most of the functions in this module have one variant with + a <c>Timeout</c> parameter, and one without. If nothing else is + specified, the default value <c>infinity</c> is used when + the <c>Timeout</c> parameter is not given.</p> - <tag><c>option() = {ssh, host()} | {port, </c> - <seealso marker="kernel:inet#type-port_number"><c>inet:port_number()</c></seealso> - <c>} | {timeout, timeout()} | SshConnectOption</c></tag> - <item><marker id="type-option"/> + </description> + <datatypes> + <datatype> + <name name="client"/> + </datatype> + <datatype> + <name name="error_reason"/> + </datatype> + <datatype> + <name name="event_time"/> + </datatype> + <datatype> + <name name="handle"/> + <desc> + <p>Opaque reference for a connection to a NETCONF server or a + NETCONF session.</p> + </desc> + </datatype> + <datatype> + <name name="host"/> + </datatype> + <datatype> + <name name="netconf_db"/> + </datatype> + <datatype> + <name name="notification"/> + </datatype> + <datatype> + <name name="notification_content"/> + </datatype> + <datatype> + <name name="option"/> + <desc> <p><c>SshConnectOption</c> is any valid option to <seealso marker="ssh:ssh#connect-3"><c>ssh:connect/3,4</c></seealso>. Common options used are <c>user</c>, <c>password</c> and <c>user_dir</c>. The <c>SshConnectOptions</c> are - verfied by the SSH application.</p></item> - - <tag><c>options() = [option()]</c></tag> - <item><marker id="type-options"/> - <p>Options used for setting up an SSH connection to a NETCONF - server.</p></item> - - <tag><c>server_id() = atom()</c></tag> - <item><marker id="type-server_id"/> + verfied by the SSH application.</p> + </desc> + </datatype> + <datatype> + <name name="options"/> + <desc> + <p>Options used for setting up an SSH connection to a NETCONF + server.</p> + </desc> + </datatype> + <datatype> + <name name="server_id"/> + <desc> <p>The identity of a server, specified in a configuration - file.</p></item> - - <tag><c>simple_xml() = {xml_tag(), xml_attributes(), xml_content()} | {xml_tag(), xml_content()} | xml_tag()</c></tag> - <item><marker id="type-simple_xml"/> - <p>This type is further described in application - <seealso marker="xmerl:index"><c>xmerl</c></seealso>.</p></item> - - <tag><c>stream_data() = {description, string()} | {replaySupport, string()} | {replayLogCreationTime, string()} | {replayLogAgedTime, string()}</c></tag> - <item><marker id="type-stream_data"/> - <p>For details about the data format for the string values, see - "XML Schema for Event Notifications" in RFC 5277.</p></item> - - <tag><c>stream_name() = string()</c></tag> - <item><marker id="type-stream_name"/> </item> - - <tag><c>streams() = [{stream_name(), [stream_data()]}]</c></tag> - <item><marker id="type-streams"/> </item> - - <tag><c>xml_attribute_tag() = atom()</c></tag> - <item><marker id="type-xml_attribute_tag"/> </item> - - <tag><c>xml_attribute_value() = string()</c></tag> - <item><marker id="type-xml_attribute_value"/> </item> - - <tag><c>xml_attributes() = [{xml_attribute_tag(), xml_attribute_value()}]</c></tag> - <item><marker id="type-xml_attributes"/> </item> - - <tag><c>xml_content() = [simple_xml() | iolist()]</c></tag> - <item><marker id="type-xml_content"/> </item> - - <tag><c>xml_tag() = atom()</c></tag> - <item><marker id="type-xml_tag"/> </item> - - <tag><c>xpath() = {xpath, string()}</c></tag> - <item><marker id="type-xpath"/> </item> - - <tag><c>xs_datetime() = string()</c></tag> - <item><marker id="type-xs_datetime"/> - <p>This date and time identifier has the same format as the XML type + file.</p> + </desc> + </datatype> + <datatype> + <name name="simple_xml"/> + <desc> + <p>This type is further described in application + <seealso marker="xmerl:index"><c>xmerl</c></seealso>.</p> + </desc> + </datatype> + <datatype> + <name name="stream_data"/> + <desc> + <p>For details about the data format for the string values, see + "XML Schema for Event Notifications" in RFC 5277.</p> + </desc> + </datatype> + <datatype> + <name name="stream_name"/> + </datatype> + <datatype> + <name name="streams"/> + </datatype> + <datatype> + <name name="xml_attribute_tag"/> + </datatype> + <datatype> + <name name="xml_attribute_value"/> + </datatype> + <datatype> + <name name="xml_attributes"/> + </datatype> + <datatype> + <name name="xml_content"/> + </datatype> + <datatype> + <name name="xml_tag"/> + </datatype> + <datatype> + <name name="xpath"/> + </datatype> + <datatype> + <name name="xs_datetime"/> + <desc> + <p>This date and time identifier has the same format as the XML type <c>dateTime</c> and is compliant with RFC 3339 Date and Time on the Internet Timestamps. The format is as follows:</p> - <pre> + <pre> [-]CCYY-MM-DDThh:mm:ss[.s][Z|(+|-)hh:mm]</pre> - </item> - </taglist> - </section> - - <funcs> - <func> - <name>action(Client, Action) -> Result</name> - <fsummary>Equivalent to action(Client, Action, infinity).</fsummary> - <desc><marker id="action-2"/> - <p>Equivalent to - <seealso marker="#action-3"><c>ct_netconfc:action(Client, Action, - infinity)</c></seealso>.</p> </desc> - </func> + </datatype> + </datatypes> + <funcs> <func> - <name>action(Client, Action, Timeout) -> Result</name> + <name name="action" arity="2"/> + <name name="action" arity="3"/> <fsummary>Executes an action.</fsummary> - <type> - <v>Client = client()</v> - <v>Action = simple_xml()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {ok, [simple_xml()]} | {error, error_reason()}</v> - </type> - <desc><marker id="action-3"/> + <desc> <p>Executes an action. If the return type is void, <c>ok</c> is returned instead of <c>{ok,[simple_xml()]}</c>.</p> </desc> </func> <func> - <name>close_session(Client) -> Result</name> - <fsummary>Equivalent to close_session(Client, infinity).</fsummary> - <desc><marker id="close_session-1"/> - <p>Equivalent to - <seealso marker="#close_session-2"><c>ct_netconfc:close_session(Client, - infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>close_session(Client, Timeout) -> Result</name> + <name name="close_session" arity="1"/> + <name name="close_session" arity="2"/> <fsummary>Requests graceful termination of the session associated with the client.</fsummary> - <type> - <v>Client = client()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="close_session-2"/> + <desc> <p>Requests graceful termination of the session associated with the client.</p> @@ -343,115 +339,148 @@ </func> <func> - <name>copy_config(Client, Source, Target) -> Result</name> - <fsummary>Equivalent to copy_config(Client, Source, Target, - infinity).</fsummary> - <desc><marker id="copy_config-3"/> - <p>Equivalent to - <seealso marker="#copy_config-4"><c>ct_netconfc:copy_config(Client, - Source, Target, infinity)</c></seealso>.</p> - </desc> - </func> + <name name="connect" arity="1"/> + <fsummary>Opens an SSH connection to a NETCONF server.</fsummary> + <desc> + <p>Opens an SSH connection to a NETCONF server.</p> - <func> - <name>copy_config(Client, Target, Source, Timeout) -> Result</name> - <fsummary>Copies configuration data.</fsummary> - <type> - <v>Client = client()</v> - <v>Target = netconf_db()</v> - <v>Source = netconf_db()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="copy_config-4"/> - <p>Copies configuration data.</p> + <p>If the server options are specified in a configuration file, use + <seealso marker="#connect-2"><c>connect/2</c></seealso> + instead.</p> - <p>Which source and target options that can be issued depends on the - capabilities supported by the server. That is, <c>:candidate</c> - and/or <c>:startup</c> are required.</p> + <p>The opaque <seealso marker="#type-handle"><c>handle()</c></seealso> + reference returned from this + function is required as connection identifier when opening + sessions over this connection, see + <seealso marker="#session-1"><c>session/1,2,3</c></seealso>.</p> + + <p>Option <c>timeout</c> (milliseconds) is used when setting up the + SSH connection. It is not used for any other purposes during the + lifetime of the connection.</p> </desc> </func> <func> - <name>create_subscription(Client) -> term()</name> - <fsummary>Creates a subscription for event notifications.</fsummary> - <desc><marker id="create_subscription-1"/></desc> - </func> + <name name="connect" arity="2"/> + <fsummary>Opens an SSH connection to a named NETCONF server.</fsummary> + <desc> + <p>Open an SSH connection to a named NETCONF server.</p> - <func> - <name>create_subscription(Client, Timeout) -> term()</name> - <fsummary>Creates a subscription for event notifications.</fsummary> - <desc><marker id="create_subscription-2"/></desc> - </func> + <p>If <c><anno>KeyOrName</anno></c> is a + configured <c>server_id()</c> or a + <c>target_name()</c> associated with such an Id, then the options + for this server are fetched from the configuration file.</p> - <func> - <name>create_subscription(Client, Stream, Timeout) -> term()</name> - <fsummary>Creates a subscription for event notifications.</fsummary> - <desc><marker id="create_subscription-3"/></desc> - </func> + <p>Argument <c><anno>ExtraOptions</anno></c> is added to the + options found in the configuration file. If the same options + are specified, the values from the configuration file + overwrite <c><anno>ExtraOptions</anno></c>.</p> - <func> - <name>create_subscription(Client, StartTime, StopTime, Timeout) -> term()</name> - <fsummary>Creates a subscription for event notifications.</fsummary> - <desc><marker id="create_subscription-4"/></desc> + <p>If the server is not specified in a configuration file, use + <seealso marker="#connect-1"><c>connect/1</c></seealso> + instead.</p> + + <p>The opaque <seealso marker="#type-handle"><c>handle()</c></seealso> + reference returned from this + function can be used as connection identifier when opening + sessions over this connection, see + <seealso marker="#session-1"><c>session/1,2,3</c></seealso>. + However, if <c><anno>KeyOrName</anno></c> is a + <c>target_name()</c>, that is, if the server is named through a + call to <seealso marker="ct#require-2"><c>ct:require/2</c></seealso> + or a <c>require</c> statement in the test suite, then this name can + be used instead of + <seealso marker="#type-handle"><c>handle()</c></seealso>.</p> + + <p>Option <c>timeout</c> (milliseconds) is used when setting up the + SSH connection. It is not used for any other purposes during the + lifetime of the connection.</p> + </desc> </func> <func> - <name>create_subscription(Client, Stream, StartTime, StopTime, Timeout) -> term()</name> - <fsummary>Creates a subscription for event notifications.</fsummary> - <desc><marker id="create_subscription-5"/></desc> + <name name="copy_config" arity="3"/> + <name name="copy_config" arity="4"/> + <fsummary>Copies configuration data.</fsummary> + <desc> + <p>Copies configuration data.</p> + + <p>Which source and target options that can be issued depends on the + capabilities supported by the server. That is, <c>:candidate</c> + and/or <c>:startup</c> are required.</p> + </desc> </func> <func> - <name>create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout) -> Result</name> + <name>create_subscription(Client) -> Result</name> + <name>create_subscription(Client, Stream) -> Result</name> + <name>create_subscription(Client, Stream, Filter) -> Result</name> + <name>create_subscription(Client, Stream, Filter, Timeout) -> Result</name> + <name name="create_subscription" arity="5" clause_i="2"/> + <name name="create_subscription" arity="6"/> <fsummary>Creates a subscription for event notifications.</fsummary> - <type> - <v>Client = client()</v> - <v>Stream = stream_name()</v> - <v>Filter = simple_xml() | [simple_xml()]</v> - <v>StartTime = xs_datetime()</v> - <v>StopTime = xs_datetime()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="create_subscription-6"/> + <desc> <p>Creates a subscription for event notifications.</p> <p>This function sets up a subscription for NETCONF event notifications of the specified stream type, matching the specified filter. The calling process receives notifications as messages of - type <c>notification()</c>.</p> + type <seealso marker="#type-notification"><c>notification()</c></seealso>.</p> + + <p>Only a subset of the function clauses are show above. The + full set of valid combinations of input parameters is as + follows:</p> + +<pre>create_subscription(Client) + +create_subscription(Client, Timeout) +create_subscription(Client, Stream) +create_subscription(Client, Filter) + +create_subscription(Client, Stream, Timeout) +create_subscription(Client, Filter, Timeout) +create_subscription(Client, Stream, Filter) +create_subscription(Client, StartTime, StopTime) + +create_subscription(Client, Stream, Filter, Timeout) +create_subscription(Client, StartTime, StopTime, Timeout) +create_subscription(Client, Stream, StartTime, StopTime) +create_subscription(Client, Filter, StartTime, StopTime) + +create_subscription(Client, Stream, StartTime, StopTime, Timeout) +create_subscription(Client, Stream, Filter, StartTime, StopTime) +create_subscription(Client, Stream, Filter, StartTime, StopTime, Timeout)</pre> <taglist> - <tag><c>Stream</c></tag> + <tag><c><anno>Stream</anno></c></tag> <item><p>Optional parameter that indicates which stream of event is of interest. If not present, events in the default NETCONF stream are sent.</p></item> - <tag><c>Filter</c></tag> + <tag><c><anno>Filter</anno></c></tag> <item><p>Optional parameter that indicates which subset of all possible events is of interest. The parameter format is the same as that of the filter parameter in the NETCONF protocol operations. If not present, all events not precluded by other parameters are sent.</p></item> - <tag><c>StartTime</c></tag> + <tag><c><anno>StartTime</anno></c></tag> <item><p>Optional parameter used to trigger the replay feature and indicate that the replay is to start at the time specified. - If <c>StartTime</c> is not present, this is not a replay - subscription.</p> + If <c><anno>StartTime</anno></c> is not present, this is not a + replay subscription.</p> <p>It is not valid to specify start times that are later than - the current time. If <c>StartTime</c> is specified earlier - than the log can support, the replay begins with the earliest - available notification.</p> + the current time. If <c><anno>StartTime</anno></c> is specified + earlier than the log can support, the replay begins with the + earliest available notification.</p> <p>This parameter is of type <c>dateTime</c> and compliant to RFC 3339. Implementations must support time zones.</p></item> - <tag><c>StopTime</c></tag> + <tag><c><anno>StopTime</anno></c></tag> <item><p>Optional parameter used with the optional replay feature to indicate the newest notifications of interest. If - <c>StopTime</c> is not present, the notifications continues - until the subscription is terminated.</p> + <c><anno>StopTime</anno></c> is not present, the notifications + continues until the subscription is terminated.</p> <p>Must be used with and be later than <c>StartTime</c>. Values - of <c>StopTime</c> in the future are valid. This parameter is - of type <c>dateTime</c> and compliant to RFC 3339. + of <c><anno>StopTime</anno></c> in the future are valid. This + parameter is of type <c>dateTime</c> and compliant to RFC 3339. Implementations must support time zones.</p></item> </taglist> @@ -461,25 +490,10 @@ </func> <func> - <name>delete_config(Client, Target) -> Result</name> - <fsummary>Equivalent to delete_config(Client, Target, - infinity).</fsummary> - <desc><marker id="delete_config-2"/> - <p>Equivalent to - <seealso marker="#delete_config-3"><c>ct_netconfc:delete_config(Client, Target, infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>delete_config(Client, Target, Timeout) -> Result</name> + <name name="delete_config" arity="2"/> + <name name="delete_config" arity="3"/> <fsummary>Deletes configuration data.</fsummary> - <type> - <v>Client = client()</v> - <v>Target = startup | candidate</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="delete_config-3"/> + <desc> <p>Deletes configuration data.</p> <p>The running configuration cannot be deleted and <c>:candidate</c> @@ -487,54 +501,25 @@ </desc> </func> - <func> - <name>edit_config(Client, Target, Config) -> Result</name> - <fsummary>Equivalent to edit_config(Client, Target, Config, [], - infinity).</fsummary> - <desc><marker id="edit_config-3"/> - <p>Equivalent to - <seealso marker="#edit_config-5"><c>ct_netconfc:edit_config(Client, - Target, Config, [], infinity)</c></seealso>.</p> - </desc> - </func> + <func> + <name name="disconnect" arity="1"/> + <fsummary>Closes the given SSH connection.</fsummary> + <desc> + <p>Closes the given SSH connection.</p> - <func> - <name>edit_config(Client, Target, Config, OptParamsOrTimeout) -> Result</name> - <fsummary>If OptParamsOrTimeout is a time-out value, this function is - equivalent to ct_netconfc:edit_config(Client, Target, Config, [], - Timeout).</fsummary> - <type> - <v>Client = client()</v> - <v>Target = netconf_db()</v> - <v>Config = simple_xml()</v> - <v>OptParamsOrTimeout = [simple_xml()] | timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="edit_config-4"/> - <p>If <c>OptParamsOrTimeout</c> is a time-out value, this function is - equivalent to - <seealso marker="#edit_config-5"><c>ct_netconfc:edit_config(Client, - Target, Config, [], Timeout)</c></seealso>.</p> - - <p>If <c>OptParamsOrTimeout</c> is a list of simple XML, this - function is equivalent to - <seealso marker="#edit_config-5"><c>ct_netconfc:edit_config(Client, - Target, Config, OptParams, infinity)</c></seealso>.</p> + <p>If there are open NETCONF sessions on the connection, these + will be brutally aborted. To avoid this, close each session + with <seealso marker="#close_session-1"><c>close_session/1,2</c></seealso></p> </desc> </func> <func> - <name>edit_config(Client, Target, Config, OptParams, Timeout) -> Result</name> + <name name="edit_config" arity="3"/> + <name name="edit_config" arity="4" clause_i="1"/> + <name name="edit_config" arity="4" clause_i="2"/> + <name name="edit_config" arity="5"/> <fsummary>Edits configuration data.</fsummary> - <type> - <v>Client = client()</v> - <v>Target = netconf_db()</v> - <v>Config = simple_xml()</v> - <v>OptParams = [simple_xml()]</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="edit_config-5"/> + <desc> <p>Edits configuration data.</p> <p>By default only the running target is available, unless the server @@ -550,29 +535,17 @@ <pre> [{'default-operation', ["none"]}, {'error-option', ["rollback-on-error"]}]</pre> - </desc> - </func> - <func> - <name>get(Client, Filter) -> Result</name> - <fsummary>Equivalent to get(Client, Filter, infinity).</fsummary> - <desc><marker id="get-2"/> - <p>Equivalent to - <seealso marker="#get-3"><c>ct_netconfc:get(Client, Filter, - infinity)</c></seealso>.</p> + <p>If <c><anno>OptParams</anno></c> is not given, the default + value <c>[]</c> is used.</p> </desc> </func> <func> - <name>get(Client, Filter, Timeout) -> Result</name> + <name name="get" arity="2"/> + <name name="get" arity="3"/> <fsummary>Gets data.</fsummary> - <type> - <v>Client = client()</v> - <v>Filter = simple_xml() | xpath()</v> - <v>Timeout = timeout()</v> - <v>Result = {ok, [simple_xml()]} | {error, error_reason()}</v> - </type> - <desc><marker id="get-3"/> + <desc> <p>Gets data.</p> <p>This operation returns both configuration and state data from the @@ -584,24 +557,10 @@ </func> <func> - <name>get_capabilities(Client) -> Result</name> - <fsummary>Equivalent to get_capabilities(Client, infinity).</fsummary> - <desc><marker id="get_capabilities-1"/> - <p>Equivalent to - <seealso marker="#get_capabilities-2"><c>ct_netconfc:get_capabilities(Client, - infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>get_capabilities(Client, Timeout) -> Result</name> + <name name="get_capabilities" arity="1"/> + <name name="get_capabilities" arity="2"/> <fsummary>Returns the server side capabilities.</fsummary> - <type> - <v>Client = client()</v> - <v>Timeout = timeout()</v> - <v>Result = [string()] | {error, error_reason()}</v> - </type> - <desc><marker id="get_capabilities-2"/> + <desc> <p>Returns the server side capabilities.</p> <p>The following capability identifiers, defined in RFC 4741 NETCONF @@ -623,26 +582,10 @@ </func> <func> - <name>get_config(Client, Source, Filter) -> Result</name> - <fsummary>Equivalent to get_config(Client, Source, Filter, - infinity).</fsummary> - <desc><marker id="get_config-3"/> - <p>Equivalent to - <seealso marker="#get_config-4"><c>ct_netconfc:get_config(Client, Source, Filter, infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>get_config(Client, Source, Filter, Timeout) -> Result</name> + <name name="get_config" arity="3"/> + <name name="get_config" arity="4"/> <fsummary>Gets configuration data.</fsummary> - <type> - <v>Client = client()</v> - <v>Source = netconf_db()</v> - <v>Filter = simple_xml() | xpath()</v> - <v>Timeout = timeout()</v> - <v>Result = {ok, [simple_xml()]} | {error, error_reason()}</v> - </type> - <desc><marker id="get_config-4"/> + <desc> <p>Gets configuration data.</p> <p>To be able to access another source than <c>running</c>, the @@ -654,25 +597,12 @@ </func> <func> - <name>get_event_streams(Client, Timeout) -> Result</name> - <fsummary>Equivalent to get_event_streams(Client, [], Timeout).</fsummary> - <desc><marker id="get_event_streams-2"/> - <p>Equivalent to - <seealso marker="#get_event_streams-3"><c>ct_netconfc:get_event_streams(Client, - [], Timeout)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>get_event_streams(Client, Streams, Timeout) -> Result</name> + <name name="get_event_streams" arity="1"/> + <name name="get_event_streams" arity="2" clause_i="1"/> + <name name="get_event_streams" arity="2" clause_i="2"/> + <name name="get_event_streams" arity="3"/> <fsummary>Sends a request to get the specified event streams.</fsummary> - <type> - <v>Client = client()</v> - <v>Streams = [stream_name()]</v> - <v>Timeout = timeout()</v> - <v>Result = {ok, streams()} | {error, error_reason()}</v> - </type> - <desc><marker id="get_event_streams-3"/> + <desc> <p>Sends a request to get the specified event streams.</p> <p><c>Streams</c> is a list of stream names. The following filter is @@ -700,67 +630,28 @@ </netconf></pre> <p>If more complex filtering is needed, use - <seealso marker="#get-2"><c>ct_netconfc:get/2</c></seealso> or - <seealso marker="#get-3"><c>ct_netconfc:get/3</c></seealso> and + <seealso marker="#get-2"><c>ct_netconfc:get/2,3</c></seealso> and specify the exact filter according to "XML Schema for Event Notifications" in RFC 5277.</p> </desc> </func> <func> - <name>get_session_id(Client) -> Result</name> - <fsummary>Equivalent to get_session_id(Client, infinity).</fsummary> - <desc><marker id="get_session_id-1"/> - <p>Equivalent to - <seealso marker="#get_session_id-2"><c>ct_netconfc:get_session_id(Client, - infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>get_session_id(Client, Timeout) -> Result</name> + <name name="get_session_id" arity="1"/> + <name name="get_session_id" arity="2"/> <fsummary>Returns the session Id associated with the specified client.</fsummary> - <type> - <v>Client = client()</v> - <v>Timeout = timeout()</v> - <v>Result = pos_integer() | {error, error_reason()}</v> - </type> - <desc><marker id="get_session_id-2"/> + <desc> <p>Returns the session Id associated with the specified client.</p> </desc> </func> <func> - <name>hello(Client) -> Result</name> - <fsummary>Equivalent to hello(Client, [], infinity).</fsummary> - <desc><marker id="hello-1"/> - <p>Equivalent to - <seealso marker="#hello-3"><c>ct_netconfc:hello(Client, [], - infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>hello(Client, Timeout) -> Result</name> - <fsummary>Equivalent to hello(Client, [], Timeout).</fsummary> - <desc><marker id="hello-2"/> - <p>Equivalent to - <seealso marker="#hello-3"><c>ct_netconfc:hello(Client, [], - Timeout)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>hello(Client, Options, Timeout) -> Result</name> + <name name="hello" arity="1"/> + <name name="hello" arity="2"/> + <name name="hello" arity="3"/> <fsummary>Exchanges hello messages with the server.</fsummary> - <type> - <v>Client = handle()</v> - <v>Options = [{capability, [string()]}]</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="hello-3"/> + <desc> <p>Exchanges <c>hello</c> messages with the server.</p> <p>Adds optional capabilities and sends a <c>hello</c> message to the @@ -769,27 +660,11 @@ </func> <func> - <name>kill_session(Client, SessionId) -> Result</name> - <fsummary>Equivalent to kill_session(Client, SessionId, - infinity).</fsummary> - <desc><marker id="kill_session-2"/> - <p>Equivalent to - <seealso marker="#kill_session-3"><c>ct_netconfc:kill_session(Client, -SessionId, infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>kill_session(Client, SessionId, Timeout) -> Result</name> + <name name="kill_session" arity="2"/> + <name name="kill_session" arity="3"/> <fsummary>Forces termination of the session associated with the supplied session Id.</fsummary> - <type> - <v>Client = client()</v> - <v>SessionId = pos_integer()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="kill_session-3"/> + <desc> <p>Forces termination of the session associated with the supplied session Id.</p> @@ -807,26 +682,11 @@ SessionId, infinity)</c></seealso>.</p> </func> <func> - <name>lock(Client, Target) -> Result</name> - <fsummary>Equivalent to lock(Client, Target, infinity).</fsummary> - <desc><marker id="lock-2"/> - <p>Equivalent to - <seealso marker="#lock-3"><c>ct_netconfc:lock(Client, Target, - infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>lock(Client, Target, Timeout) -> Result</name> - <fsummary>Unlocks the configuration target.</fsummary> - <type> - <v>Client = client()</v> - <v>Target = netconf_db()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="lock-3"/> - <p>Unlocks the configuration target.</p> + <name name="lock" arity="2"/> + <name name="lock" arity="3"/> + <fsummary>Locks the configuration target.</fsummary> + <desc> + <p>Locks the configuration target.</p> <p>Which target parameters that can be used depends on if <c>:candidate</c> and/or <c>:startup</c> are supported by the @@ -835,9 +695,7 @@ SessionId, infinity)</c></seealso>.</p> Locks are intended to be short-lived.</p> <p>Operation - <seealso marker="#kill_session-2"><c>ct_netconfc:kill_session/2</c></seealso> - or - <seealso marker="#kill_session-3"><c>ct_netconfc:kill_session/3</c></seealso> + <seealso marker="#kill_session-2"><c>kill_session/2,3</c></seealso> can be used to force the release of a lock owned by another NETCONF session. How this is achieved by the server side is implementation-specific.</p> @@ -845,54 +703,41 @@ SessionId, infinity)</c></seealso>.</p> </func> <func> - <name>only_open(Options) -> Result</name> + <name name="only_open" arity="1"/> <fsummary>Opens a NETCONF session, but does not send hello.</fsummary> - <type> - <v>Options = options()</v> - <v>Result = {ok, handle()} | {error, error_reason()}</v> - </type> - <desc><marker id="only_open-1"/> + <desc> <p>Opens a NETCONF session, but does not send <c>hello</c>.</p> - <p>As <seealso marker="#open-1"><c>ct_netconfc:open/1</c></seealso>, - but does not send a <c>hello</c> message.</p> + <p>As <seealso marker="#open-1"><c>open/1</c></seealso>, but + does not send a <c>hello</c> message.</p> </desc> </func> <func> - <name>only_open(KeyOrName, ExtraOptions) -> Result</name> - <fsummary>Opens a name NETCONF session, but does not send - hello.</fsummary> - <type> - <v>KeyOrName = key_or_name()</v> - <v>ExtraOptions = options()</v> - <v>Result = {ok, handle()} | {error, error_reason()}</v> - </type> - <desc><marker id="only_open-2"/> - <p>Opens a name NETCONF session, but does not send <c>hello</c>.</p> - - <p>As <seealso marker="#open-2"><c>ct_netconfc:open/2</c></seealso>, - but does not send a <c>hello</c> message.</p> + <name name="only_open" arity="2"/> + <fsummary>Opens a named NETCONF session, but does not send hello.</fsummary> + <desc> + <p>Opens a named NETCONF session, but does not send <c>hello</c>.</p> + + <p>As <seealso marker="#open-2"><c>open/2</c></seealso>, but + does not send a <c>hello</c> message.</p> </desc> </func> <func> - <name>open(Options) -> Result</name> - <fsummary>Opens a NETCONF session and exchanges hello messages.</fsummary> - <type> - <v>Options = options()</v> - <v>Result = {ok, handle()} | {error, error_reason()}</v> - </type> - <desc><marker id="open-1"/> + <name name="open" arity="1"/> + <fsummary>Opens a NETCONF session and exchanges hello messages.</fsummary> + <desc> <p>Opens a NETCONF session and exchanges <c>hello</c> messages.</p> <p>If the server options are specified in a configuration file, or if a named client is needed for logging purposes (see section <seealso marker="#Logging">Logging</seealso> in this module), use - <seealso marker="#open-2"><c>ct_netconfc:open/2</c></seealso> + <seealso marker="#open-2"><c>open/2</c></seealso> instead.</p> - <p>The opaque <c>handle()</c> reference returned from this + <p>The opaque <seealso marker="#type-handle"><c>handle()</c></seealso> + reference returned from this function is required as client identifier when calling any other function in this module.</p> @@ -904,37 +749,36 @@ SessionId, infinity)</c></seealso>.</p> </func> <func> - <name>open(KeyOrName, ExtraOptions) -> Result</name> + <name name="open" arity="2"/> <fsummary>Opens a named NETCONF session and exchanges hello messages.</fsummary> - <type> - <v>KeyOrName = key_or_name()</v> - <v>ExtraOptions = options()</v> - <v>Result = {ok, handle()} | {error, error_reason()}</v> - </type> - <desc><marker id="open-2"/> + <desc> <p>Opens a named NETCONF session and exchanges <c>hello</c> messages.</p> - <p>If <c>KeyOrName</c> is a configured <c>server_id()</c> or a + <p>If <c><anno>KeyOrName</anno></c> is a + configured <c>server_id()</c> or a <c>target_name()</c> associated with such an Id, then the options for this server are fetched from the configuration file.</p> - <p>Argument <c>ExtraOptions</c> is added to the options found in the - configuration file. If the same options are specified, the values - from the configuration file overwrite <c>ExtraOptions</c>.</p> + <p>Argument <c><anno>ExtraOptions</anno></c> is added to the + options found in the configuration file. If the same + options are specified, the values from the configuration + file overwrite <c><anno>ExtraOptions</anno></c>.</p> <p>If the server is not specified in a configuration file, use - <seealso marker="#open-1"><c>ct_netconfc:open/1</c></seealso> + <seealso marker="#open-1"><c>open/1</c></seealso> instead.</p> - <p>The opaque <c>handle()</c> reference returned from this + <p>The opaque <seealso marker="#type-handle"><c>handle()</c></seealso> + reference returned from this function can be used as client identifier when calling any other - function in this module. However, if <c>KeyOrName</c> is a + function in this module. However, if <c><anno>KeyOrName</anno></c> is a <c>target_name()</c>, that is, if the server is named through a call to <seealso marker="ct#require-2"><c>ct:require/2</c></seealso> or a <c>require</c> statement in the test suite, then this name can - be used instead of <c>handle()</c>.</p> + be used instead of + <seealso marker="#type-handle"><c>handle()</c></seealso>.</p> <p>Option <c>timeout</c> (milliseconds) is used when setting up the SSH connection and when waiting for the <c>hello</c> message from @@ -947,25 +791,10 @@ SessionId, infinity)</c></seealso>.</p> </func> <func> - <name>send(Client, SimpleXml) -> Result</name> - <fsummary>Equivalent to send(Client, SimpleXml, infinity).</fsummary> - <desc><marker id="send-2"/> - <p>Equivalent to - <seealso marker="#send-3"><c>ct_netconfc:send(Client, SimpleXml, - infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>send(Client, SimpleXml, Timeout) -> Result</name> + <name name="send" arity="2"/> + <name name="send" arity="3"/> <fsummary>Sends an XML document to the server.</fsummary> - <type> - <v>Client = client()</v> - <v>SimpleXml = simple_xml()</v> - <v>Timeout = timeout()</v> - <v>Result = simple_xml() | {error, error_reason()}</v> - </type> - <desc><marker id="send-3"/> + <desc> <p>Sends an XML document to the server.</p> <p>The specified XML document is sent "as is" to the server. This @@ -975,25 +804,10 @@ SessionId, infinity)</c></seealso>.</p> </func> <func> - <name>send_rpc(Client, SimpleXml) -> Result</name> - <fsummary>Equivalent to send_rpc(Client, SimpleXml, infinity).</fsummary> - <desc><marker id="send_rpc-2"/> - <p>Equivalent to - <seealso marker="#send_rpc-3"><c>ct_netconfc:send_rpc(Client, - SimpleXml, infinity)</c></seealso>.</p> - </desc> - </func> - - <func> - <name>send_rpc(Client, SimpleXml, Timeout) -> Result</name> + <name name="send_rpc" arity="2"/> + <name name="send_rpc" arity="3"/> <fsummary>Sends a NETCONF rpc request to the server.</fsummary> - <type> - <v>Client = client()</v> - <v>SimpleXml = simple_xml()</v> - <v>Timeout = timeout()</v> - <v>Result = [simple_xml()] | {error, error_reason()}</v> - </type> - <desc><marker id="send_rpc-3"/> + <desc> <p>Sends a NETCONF <c>rpc</c> request to the server.</p> <p>The specified XML document is wrapped in a valid NETCONF <c>rpc</c> @@ -1006,30 +820,42 @@ SessionId, infinity)</c></seealso>.</p> </func> <func> - <name>unlock(Client, Target) -> Result</name> - <fsummary>Equivalent to unlock(Client, Target, infinity).</fsummary> - <desc><marker id="unlock-2"/> - <p>Equivalent to - <seealso marker="#unlock-3"><c>ct_netconfc:unlock(Client, Target, - infinity)</c></seealso>.</p> + <name name="session" arity="1"/> + <name name="session" arity="2" clause_i="1"/> + <name name="session" arity="2" clause_i="2"/> + <name name="session" arity="3"/> + <fsummary>Opens a NETCONF session as a channel on the given SSH + connection, and exchanges hello messages with the + server.</fsummary> + <type name="session_options"/> + <type name="session_option"/> + <desc> + <p>Opens a NETCONF session as a channel on the given SSH + connection, and exchanges hello messages with the server.</p> + + <p>The opaque <seealso marker="#type-handle"><c>handle()</c></seealso> + reference returned from this + function can be used as client identifier when calling any + other function in this module. However, if <c><anno>KeyOrName</anno></c> + is used and it is a <c>target_name()</c>, that is, if the + server is named through a call + to <seealso marker="ct#require-2"><c>ct:require/2</c></seealso> + or a <c>require</c> statement in the test suite, then this + name can be used instead of + <seealso marker="#type-handle"><c>handle()</c></seealso>.</p> + </desc> </func> <func> - <name>unlock(Client, Target, Timeout) -> Result</name> + <name name="unlock" arity="2"/> + <name name="unlock" arity="3"/> <fsummary>Unlocks the configuration target.</fsummary> - <type> - <v>Client = client()</v> - <v>Target = netconf_db()</v> - <v>Timeout = timeout()</v> - <v>Result = ok | {error, error_reason()}</v> - </type> - <desc><marker id="unlock-3"/> + <desc> <p>Unlocks the configuration target.</p> <p>If the client earlier has acquired a lock through - <seealso marker="#lock-2"><c>ct_netconfc:lock/2</c></seealso> or - <seealso marker="#lock-3"><c>ct_netconfc:lock/3</c></seealso>, this + <seealso marker="#lock-2"><c>lock/2,3</c></seealso>, this operation releases the associated lock. To access another target than <c>running</c>, the server must support <c>:candidate</c> and/or <c>:startup</c>.</p> diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 9e6229f1dd..5b962ed5c7 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>2007</year><year>2016</year> + <year>2007</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -125,6 +125,7 @@ [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]] [-basic_html] [-no_esc_chars] + [-keep_logs all | NLogs] [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. CTHModuleN CTHOptsN] [-exit_status ignore_config] @@ -164,6 +165,7 @@ [-until [YYMoMoDD]HHMMSS [-force_stop [skip_rest]]] [-basic_html] [-no_esc_chars] + [-keep_logs all | NLogs] [-ct_hooks CTHModule1 CTHOpts1 and CTHModule2 CTHOpts2 and .. CTHModuleN CTHOptsN] [-exit_status ignore_config]</pre> @@ -189,13 +191,15 @@ [-scale_timetraps] [-create_priv_dir auto_per_run | auto_per_tc | manual_per_tc] [-basic_html] - [-no_esc_chars]</pre> + [-no_esc_chars] + [-keep_logs all | NLogs]</pre> </section> <section> <title>Refresh HTML Index Files</title> <pre> - ct_run -refresh_logs [-logdir LogDir] [-basic_html]</pre> + ct_run -refresh_logs [-logdir LogDir] [-basic_html] + [-keep_logs all | NLogs]</pre> </section> <section> diff --git a/lib/common_test/doc/src/ct_ssh.xml b/lib/common_test/doc/src/ct_ssh.xml index 137e4c3f1d..63627d8840 100644 --- a/lib/common_test/doc/src/ct_ssh.xml +++ b/lib/common_test/doc/src/ct_ssh.xml @@ -1034,6 +1034,33 @@ ChannelId, 0, Data, End, Timeout)</c></seealso>.</p> </func> <func> + <name>shell(SSH, ChannelId) -> ok | {error, Reason}</name> + <fsummary>Equivalent to shell(SSH, ChannelId, DefaultTimeout).</fsummary> + <desc><marker id="shell-2"/> + <p>Equivalent to + <seealso marker="#shell-3"><c>ct_ssh:shell(SSH, ChannelId, + DefaultTimeout)</c></seealso>.</p> + </desc> + </func> + + <func> + <name>shell(SSH, ChannelId, Timeout) -> ok | {error, Reason}</name> + <fsummary>Requests that the user default shell is executed at the + server end.</fsummary> + <type> + <v>SSH = connection()</v> + <v>ChannelId = integer()</v> + <v>Timeout = integer()</v> + <v>Reason = term()</v> + </type> + <desc><marker id="shell-3"/> + <p>Requests that the user default shell (typically defined in + <c>/etc/passwd</c> in Unix systems) is executed at the + server end.</p> + </desc> + </func> + + <func> <name>subsystem(SSH, ChannelId, Subsystem) -> Status | {error, Reason}</name> <fsummary>Equivalent to subsystem(SSH, ChannelId, Subsystem, DefaultTimeout).</fsummary> diff --git a/lib/common_test/doc/src/ct_telnet.xml b/lib/common_test/doc/src/ct_telnet.xml index eba3c3030d..8e85cccc99 100644 --- a/lib/common_test/doc/src/ct_telnet.xml +++ b/lib/common_test/doc/src/ct_telnet.xml @@ -337,7 +337,7 @@ <c>FullMatch</c> is the string matched by the whole regular expression, and <c>SubMatchN</c> is the string that matched subexpression number <c>N</c>. Subexpressions are denoted with - <c>(' ')</c> in the regular expression.</p> + <c>'(' ')'</c> in the regular expression.</p> <p>If a <c>Tag</c> is speciifed, the returned <c>Match</c> also includes the matched <c>Tag</c>. Otherwise, only <c>RxMatch</c> diff --git a/lib/common_test/doc/src/ct_testspec.xml b/lib/common_test/doc/src/ct_testspec.xml new file mode 100644 index 0000000000..36893f66cf --- /dev/null +++ b/lib/common_test/doc/src/ct_testspec.xml @@ -0,0 +1,84 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!DOCTYPE erlref SYSTEM "erlref.dtd"> + +<erlref> + <header> + <copyright> + <year>2016</year> + <holder>Ericsson AB. All Rights Reserved.</holder> + </copyright> + <legalnotice> + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + </legalnotice> + + <title>ct_testspec</title> + <prepared></prepared> + <responsible></responsible> + <docno></docno> + <approved></approved> + <checked></checked> + <date></date> + <rev>A</rev> + <file>ct_testspec.xml</file> + </header> + <module>ct_testspec</module> + <modulesummary>Parsing of test specifications for Common Test. + </modulesummary> + +<description> + + <p>Parsing of test specifications for <c>Common Test</c>.</p> + + <p>This module exports help functions for parsing of test specifications.</p> + +</description> + + <funcs> + <func> + <name>get_tests(SpecsIn) -> {ok, [{Specs,Tests}]} | {error, Reason}</name> + <fsummary>Parse the given test specification files and return the tests to run and skip.</fsummary> + <type> + <v>SpecsIn = [string()] | [[string()]]</v> + <v>Specs = [string()]</v> + <v>Test = [{Node,Run,Skip}]</v> + <v>Node = atom()</v> + <v>Run = {Dir,Suites,Cases}</v> + <v>Skip = {Dir,Suites,Comment} | {Dir,Suites,Cases,Comment}</v> + <v>Dir = string()</v> + <v>Suites = atom | [atom()] | all</v> + <v>Cases = atom | [atom()] | all</v> + <v>Comment = string()</v> + <v>Reason = term()</v> + </type> + <desc><marker id="add_nodes-1"/> + <p>Parse the given test specification files and return the + tests to run and skip.</p> + + <p>If <c>SpecsIn=[Spec1,Spec2,...]</c>, separate tests will be + created per specification. If + <c>SpecsIn=[[Spec1,Spec2,...]]</c>, all specifications will be + merge into one test.</p> + + <p>For each test, a <c>{Specs,Tests}</c> element is returned, + where <c>Specs</c> is a list of all included test + specifications, and <c>Tests</c> specifies actual tests to + run/skip per node.</p> + </desc> + </func> + + </funcs> + +</erlref> + + diff --git a/lib/common_test/doc/src/notes.xml b/lib/common_test/doc/src/notes.xml index 7653670d30..cc676e955b 100644 --- a/lib/common_test/doc/src/notes.xml +++ b/lib/common_test/doc/src/notes.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2004</year><year>2016</year> + <year>2004</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -33,6 +33,250 @@ <file>notes.xml</file> </header> +<section><title>Common_Test 1.15</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Errors in the documentation for user HTML stylesheets + have been corrected.</p> + <p> + Own Id: OTP-14332 Aux Id: seq13299 </p> + </item> + <item> + <p>Internal code change: Calls to <c>catch</c> followed + by a call to <c>erlang:get_stacktrace/0</c> has been + rewritten to use <c>try</c> instead of <c>catch</c> to + make the code future-proof.</p> + <p> + Own Id: OTP-14400</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>The <c>ct_slave</c> modules now handle nodenames in + the same way as nodenames passed to <c>-sname</c>. That + means <c>ct_slave:start('[email protected]').</c> will now + work.</p> + <p> + Own Id: OTP-13806</p> + </item> + <item> + <p> + Added the new option, <c>keep_logs</c>. If setting the + value for this option to an integer, N, common_test will + remove all ct_run.* directories in the current log + directory, except the N newest.</p> + <p> + Own Id: OTP-14179</p> + </item> + <item> + <p> + The existing <c>ct_netconfc:open/1,2</c> opens an SSH + connection with one SSH channel realizing one Netconf + session. To allow testing of multiple sessions over the + same SSH connection, the following functions are added to + <c>ct_netconfc</c>:</p> + <p> + * <c>connect/1,2</c> - establish an SSH connection * + <c>disconnect/1</c> - close the given SSH connection * + <c>session/1,2,3</c> - open an ssh channel on the given + connection and send 'hello' to start a Netconf session</p> + <p> + Own Id: OTP-14284</p> + </item> + <item> + <p> + The function ct_ssh:shell/2,3 is added.</p> + <p> + Own Id: OTP-14415 Aux Id: seq13315 </p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.14</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>The following corrections and improvements are done in + the common_test hook handling:</p> <list> <item> <p>An + extra argument, <c>Suite</c>, is added as the first + argument to each of the following hook callback + functions:</p> <list> + <item><c>pre_init_per_group</c></item> + <item><c>post_init_per_group</c></item> + <item><c>pre_end_per_group</c></item> + <item><c>post_end_per_group</c></item> + <item><c>pre_init_per_testcase</c></item> + <item><c>post_init_per_testcase</c></item> + <item><c>pre_end_per_testcase</c></item> + <item><c>post_end_per_testcase</c></item> + <item><c>on_tc_fail</c></item> + <item><c>on_tc_skip</c></item> </list> <p>For backwards + compatibility, if the new function is not exported from a + hook callback module, <c>common_test</c> will fall back + to the old interface and call the function without the + <c>Suite</c> argument.</p> </item> <item> <p>If either + <c>init_per_suite</c> or <c>end_per_suite</c> exists, but + not the other, then the non-existing function will be + reported as failed with reason <c>undef</c> in the test + log. The same goes for <c>init/end_per_group</c>. This + has always been a requirement according to the user's + guide, but now <c>common_test</c> is more explicit in the + report.</p> </item> <item> <p>If <c>init_per_suite</c> + was exported from a test suite, but not + <c>end_per_suite</c>, then <c>pre/post_end_per_suite</c> + was called with <c>Suite=ct_framework</c> instead of the + correct suite name. This is now corrected.</p> </item> + <item> <p>If <c>end_per_group</c> was exported from a + suite, but not <c>init_per_group</c>, then + <c>end_per_group</c> was never called. This is now + corrected.</p> </item> <item> <p>Tests that were skipped + before calling <c>pre_init_per_*</c> got faulty calls to + the corresponding <c>post_init_per_*</c>. E.g. if a test + was skipped because <c>suite/0</c> failed, then + <c>post_init_per_suite</c> would be called even though + <c>pre_init_per_suite</c> and <c>init_per_suite</c> were + not called. This is now corrected so a <c>post_*</c> + callback will never be called unless the corresponding + <c>pre_*</c> callback has been called first.</p> </item> + <item> <p>Tests that were skipped before or in + <c>init_per_testcase</c> got faulty calls to + <c>pre_end_per_testcase</c> and + <c>post_end_per_testcase</c>. This is now corrected so + <c>pre/post_end_per_testcase</c> are not called when + <c>end_per_testcase</c> is not called.</p> </item> <item> + <p>If an exit signal causes the test case process to die + while running <c>init_per_testcase</c>, the case was + earlier reported as failed with reason <c>{skip,...}</c>. + This is now corrected so the case will be marked as + skipped.</p> </item> <item> <p>If an exist signal causes + the test case process to die while running + <c>end_per_testcase</c>, the case was earlier marked as + failed. This is now corrected so the status of the test + case is not changed - there is only a warning added to + the comment field.</p> </item> <item> <p>If a test case + was skipped because of option + <c>{force_stop,skip_rest}</c> or because of a failed + sequence, then no <c>tc_start</c> event would be sent, + only <c>tc_done</c>. This is now corrected so both events + are sent.</p> </item> <item> <p>When skipping or failing + in a configuration function, the configuration function + itself would get <c>{auto_skipped,Reason}</c>, + <c>{skipped,Reason}</c> or <c>{failed,Reason}</c> in the + hook callbacks <c>on_tc_skip</c> or <c>on_tc_fail</c>. + The other test cases that were skipped as a result of + this would only get <c>Reason</c> in <c>on_tc_skip</c>. + This is now corrected so even the configuration function + that caused the skip/fail will only get <c>Reason</c> in + the hook callback.</p> </item> </list> + <p> + Own Id: OTP-10599 Aux Id: kunagi-344 [255] </p> + </item> + <item> + <p> + When a test case was skipped by a <c>skip_cases</c> + statement in a test spec, then <c>cth_surefire</c> would + erroneously mark the previous test case as skipped in the + xml report. The actually skipped test case would not be + present in the xml report at all. This is now corrected.</p> + <p> + Own Id: OTP-14129 Aux Id: seq13244 </p> + </item> + <item> + <p>The <c>multiply_timetraps</c> and + <c>scale_timetraps</c> options did not work with test + specifications, which has been corrected.</p> + <p> + Own Id: OTP-14210</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + ct_testspec:get_tests/1 is added. This is used by rebar3 + to get all directories that must be compiled when running + tests from testspec - instead of implementing testspec + parsing in rebar3.</p> + <p> + Own Id: OTP-14132</p> + </item> + </list> + </section> + +</section> + +<section><title>Common_Test 1.13</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Some types of printouts to screen during test runs + (including <c>ct:print/1,2,3,4</c>) used the local + <c>user</c> process as IO device and these printouts + would not be visible when e.g. running tests via a shell + on a remote node. A default Common Test group leader + process has been introduced to solve the problem. This + process routes printouts to the group leader of the + starting process, if available, otherwise to <c>user</c>.</p> + <p> + Own Id: OTP-13973 Aux Id: ERL-279 </p> + </item> + <item> + <p> + Some Common Test processes, that act as I/O group leaders + for test cases, would not terminate as expected at the + end of test runs. This error has been corrected.</p> + <p> + Own Id: OTP-14026 Aux Id: ERL-287 </p> + </item> + <item> + <p> + The logging verbosity feature was incorrectly documented. + The default verbosity levels for test runs is e.g. not 50 + (<c>?STD_VERBOSITY</c>), but 100 (<c>?MAX_VERBOSITY</c>). + Also, some of the examples had errors and flaws. The + corresponding chapter (5.18) in the User's Guide has been + updated.</p> + <p> + Own Id: OTP-14044 Aux Id: seq13223 </p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + A feature to let the user specify headings to log + printouts has been added. The heading is specified as + <c>{heading,string()}</c> in the <c>Opts</c> list + argument to <c>ct:pal/3,4,5</c>, <c>ct:print/3,4,5</c>, + or <c>ct:log/3,4,5</c>. If the heading option is omitted, + the category name, or <c>"User"</c>, is used as the + heading instead.</p> + <p> + Own Id: OTP-14043 Aux Id: seq13226 </p> + </item> + </list> + </section> + +</section> + <section><title>Common_Test 1.12.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/common_test/doc/src/ref_man.xml b/lib/common_test/doc/src/ref_man.xml index d1567e2d3c..1ac20db5c2 100644 --- a/lib/common_test/doc/src/ref_man.xml +++ b/lib/common_test/doc/src/ref_man.xml @@ -47,6 +47,7 @@ <xi:include href="ct_slave.xml"/> <xi:include href="ct_hooks.xml"/> <xi:include href="ct_property_test.xml"/> + <xi:include href="ct_testspec.xml"/> </application> diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 76e306c4ed..56f6f7bcc4 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2016</year> + <year>2003</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -1322,8 +1322,8 @@ <title>The Unexpected I/O Log</title> <p>The test suites overview page includes a link to the Unexpected I/O Log. In this log, <c>Common Test</c> saves printouts made with - <seealso marker="ct#log-2"><c>ct:log/2</c></seealso> and - <seealso marker="ct#pal-2"><c>ct:pal/2</c></seealso>, as well as captured system + <seealso marker="ct#log-2"><c>ct:log/1,2,3,4,5</c></seealso> and + <seealso marker="ct#pal-2"><c>ct:pal/1,2,3,4,5</c></seealso>, as well as captured system error- and progress reports, which cannot be associated with particular test cases and therefore cannot be written to individual test case log files. This occurs, for example, if a log printout is made from an external process (not a test @@ -1338,7 +1338,7 @@ <title>The Pre- and Post Test I/O Log</title> <p>The <c>Common Test</c> Framework Log page includes links to the Pre- and Post Test I/O Log. In this log, <c>Common Test</c> saves printouts made - with <c>ct:log/2</c> and <c>ct:pal/2</c>, as well as captured system error- + with <c>ct:log/1,2,3,4,5</c> and <c>ct:pal/1,2,3,4,5</c>, as well as captured system error- and progress reports, which take place before, and after, the test run. Examples of this are printouts from a CT hook init- or terminate function, or progress reports generated when an OTP application is started from a CT hook @@ -1349,10 +1349,22 @@ applications, see section <seealso marker="ct_hooks_chapter#synchronizing">Synchronizing</seealso> in section Common Test Hooks.</p> - <note><p>Logging to file with <c>ct:log/2</c> or <c>ct:pal/2</c> - only works when <c>Common Test</c> is running. Printouts with <c>ct:pal/2</c> + <note><p>Logging to file with <c>ct:log/1,2,3,4,5</c> or <c>ct:pal/1,2,3,4,5</c> + only works when <c>Common Test</c> is running. Printouts with <c>ct:pal/1,2,3,4,5</c> are however always displayed on screen.</p></note> </section> + + <section> + <marker id="delete_old_logs"></marker> + <title>Delete Old Logs</title> + <p><c>Common Test</c> can automatically delete old log. This + is specified with the <c>keep_logs</c> option. The default + value for this option is <c>all</c>, which means that no + logs are deleted. If the value is set to an + integer, <c>N</c>, <c>Common Test</c> deletes + all <c>ct_run.<timestamp></c> directories, except + the <c>N</c> newest.</p> + </section> </section> <section> @@ -1371,24 +1383,38 @@ <p><c>Common Test</c> includes an <em>optional</em> feature to allow user HTML style sheets for customizing printouts. The functions in <c>ct</c> that print to a test case HTML log - file (<c>log/3</c> and <c>pal/3</c>) accept <c>Category</c> + file (<c>log/3,4,5</c> and <c>pal/3,4,5</c>) accept <c>Category</c> as first argument. With this argument a category can be specified - that can be mapped to a selector in a CSS - definition. This is useful, especially for coloring text + that can be mapped to a named <c>div</c> selector in a CSS rule-set. + This is useful, especially for coloring text differently depending on the type of (or reason for) the - printout. Say you want one color for test system + printout. Say you want one particular background color for test system configuration information, a different one for test system state information, and finally one for errors detected by the test case functions. The corresponding style sheet can look as follows:</p> <pre> - div.sys_config { background:blue; color:white } - div.sys_state { background:yellow; color:black } - div.error { background:red; color:white }</pre> + div.sys_config { background:blue } + div.sys_state { background:yellow } + div.error { background:red }</pre> + + <p>Common Test prints the text from <c>ct:log/3,4,5</c> or + <c>ct:pal/3,4,5</c> inside a <c>pre</c> element + nested under the named <c>div</c> element. Since the <c>pre</c> selector + has a predefined CSS rule (in file <c>ct_default.css</c>) for the attributes + <c>color</c>, <c>font-family</c> and <c>font-size</c>, if a user wants to + change any of the predefined attribute settings, a new rule for <c>pre</c> + must be added to the user stylesheet. Example:</p> + + <pre> +div.error pre { color:white }</pre> + + <p>Here, white text is used instead of the default black for <c>div.error</c> + printouts (and no other attribute settings for <c>pre</c> are affected).</p> <p>To install the CSS file (<c>Common Test</c> inlines the definition in the - HTML code), the name can be provided when executing <c>ct_run</c>.</p> + HTML code), the file name can be provided when executing <c>ct_run</c>.</p> <p><em>Example:</em></p> <pre> @@ -1436,8 +1462,8 @@ suite.</p> <p>Argument <c>Category</c> in the previous example can have the - value (atom) <c>sys_config</c> (white on blue), <c>sys_state</c> - (black on yellow), or <c>error</c> (white on red).</p> + value (atom) <c>sys_config</c> (blue background), <c>sys_state</c> + (yellow background), or <c>error</c> (white text on red background).</p> </section> <section> diff --git a/lib/common_test/doc/src/specs.xml b/lib/common_test/doc/src/specs.xml new file mode 100644 index 0000000000..7e40e8351d --- /dev/null +++ b/lib/common_test/doc/src/specs.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="utf-8" ?> +<specs xmlns:xi="http://www.w3.org/2001/XInclude"> + <xi:include href="../specs/specs_ct_netconfc.xml"/> + <xi:include href="../specs/specs_ct.xml"/> +</specs> diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index 1d3fbb6f76..82dc06834f 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2003</year><year>2016</year> + <year>2003</year><year>2017</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -566,7 +566,7 @@ for the test cases in the group. After execution of the group is finished, function <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group(GroupName, Config)</c></seealso> is called. This function is meant to be used for cleaning up after - <c>init_per_group/2</c>.</p> + <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p> <p>Whenever a group is executed, if <c>init_per_group</c> and <c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls @@ -986,15 +986,17 @@ <c>io:put_chars/1</c>, and so on.</p> <p><c>Importance</c> is compared to a verbosity level set by the - <c>verbosity</c> start flag/option. The verbosity level can be set per - category or generally, or both. The default verbosity level, - <c>?STD_VERBOSITY</c>, is 50, that is, all standard I/O gets printed. - If a lower verbosity level is set, standard I/O printouts are ignored. - <c>Common Test</c> performs the following test:</p> + <c>verbosity</c> start flag/option. The level can be set per + category or generally, or both. If <c>verbosity</c> is not set by the user, + a level of 100 (<c>?MAX_VERBOSITY</c> = all printouts visible) is used as + default value. <c>Common Test</c> performs the following test:</p> <pre> - Importance >= (100-VerbosityLevel)</pre> - <p>This also means that verbosity level 0 effectively turns all logging off - (except from printouts made by <c>Common Test</c> itself).</p> +Importance >= (100-VerbosityLevel)</pre> + <p>The constant <c>?STD_VERBOSITY</c> has value 50 (see <c>ct.hrl</c>). + At this level, all standard I/O gets printed. If a lower verbosity level + is set, standard I/O printouts are ignored. Verbosity level 0 effectively + turns all logging off (except from printouts made by <c>Common Test</c> + itself).</p> <p>The general verbosity level is not associated with any particular category. This level sets the threshold for the standard I/O printouts, @@ -1003,17 +1005,17 @@ <p><em>Examples:</em></p> <p>Some printouts during test case execution:</p> - <pre> + <pre> io:format("1. Standard IO, importance = ~w~n", [?STD_IMPORTANCE]), ct:log("2. Uncategorized, importance = ~w", [?STD_IMPORTANCE]), - ct:log(info, "3. Categorized info, importance = ~w", [?STD_IMPORTANCE]]), + ct:log(info, "3. Categorized info, importance = ~w", [?STD_IMPORTANCE]), ct:log(info, ?LOW_IMPORTANCE, "4. Categorized info, importance = ~w", [?LOW_IMPORTANCE]), - ct:log(error, "5. Categorized error, importance = ~w", [?HI_IMPORTANCE]), - ct:log(error, ?HI_IMPORTANCE, "6. Categorized error, importance = ~w", [?MAX_IMPORTANCE]),</pre> + ct:log(error, ?HI_IMPORTANCE, "5. Categorized error, importance = ~w", [?HI_IMPORTANCE]), + ct:log(error, ?MAX_IMPORTANCE, "6. Categorized error, importance = ~w", [?MAX_IMPORTANCE]),</pre> - <p>If starting the test without specifying any verbosity levels as follows:</p> + <p>If starting the test with a general verbosity level of 50 (<c>?STD_VERBOSITY</c>):</p> <pre> - $ ct_run ...</pre> + $ ct_run -verbosity 50</pre> <p>the following is printed:</p> <pre> 1. Standard IO, importance = 50 @@ -1031,10 +1033,22 @@ 4. Categorized info, importance = 25 6. Categorized error, importance = 99</pre> + <p>Note that the category argument is not required in order to only specify the + importance of a printout. Example:</p> + <pre> +ct:pal(?LOW_IMPORTANCE, "Info report: ~p", [Info])</pre> + <p>Or perhaps in combination with constants:</p> + <pre> +-define(INFO, ?LOW_IMPORTANCE). +-define(ERROR, ?HI_IMPORTANCE). + +ct:log(?INFO, "Info report: ~p", [Info]) +ct:pal(?ERROR, "Error report: ~p", [Error])</pre> + <p>The functions <seealso marker="ct#set_verbosity-2"><c>ct:set_verbosity/2</c></seealso> and <seealso marker="ct#get_verbosity-1"><c>ct:get_verbosity/1</c></seealso> may be used to modify and read verbosity levels during test execution.</p> - + <p>The arguments <c>Format</c> and <c>FormatArgs</c> in <c>ct:log/print/pal</c> are always passed on to the STDLIB function <c>io:format/3</c> (For details, see the <seealso marker="stdlib:io"><c>io</c></seealso> manual page).</p> diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index 0f9e044f9e..9d751996ad 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -80,6 +80,7 @@ MODULES= \ ct_groups \ ct_property_test \ ct_release_test \ + ct_default_gl \ erl2html2 \ test_server_ctrl \ test_server_gl \ diff --git a/lib/common_test/src/common_test.app.src b/lib/common_test/src/common_test.app.src index 77588af59b..430a4fa2fb 100644 --- a/lib/common_test/src/common_test.app.src +++ b/lib/common_test/src/common_test.app.src @@ -1,7 +1,7 @@ % This is an -*- erlang -*- file. %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ {vsn, "%VSN%"}, {modules, [ct_cover, ct, + ct_default_gl, ct_event, ct_framework, ct_ftp, diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index f9f845e1a9..662732d475 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -66,8 +66,8 @@ reload_config/1, escape_chars/1, escape_chars/2, log/1, log/2, log/3, log/4, log/5, - print/1, print/2, print/3, print/4, - pal/1, pal/2, pal/3, pal/4, + print/1, print/2, print/3, print/4, print/5, + pal/1, pal/2, pal/3, pal/4, pal/5, set_verbosity/2, get_verbosity/1, capture_start/0, capture_stop/0, capture_get/0, capture_get/1, fail/1, fail/2, comment/1, comment/2, make_priv_dir/0, @@ -89,6 +89,36 @@ -export([get_target_name/1]). -export([parse_table/1, listenv/1]). +%%---------------------------------------------------------------------- +%% Exported types +%%---------------------------------------------------------------------- +%% For ct_gen_conn +-export_type([config_key/0, + target_name/0, + key_or_name/0]). + +%% For cth_conn_log +-export_type([conn_log_options/0, + conn_log_type/0, + conn_log_mod/0]). + +%%------------------------------------------------------------------ +%% Type declarations +%% ------------------------------------------------------------------ +-type config_key() :: atom(). % Config key which exists in a config file +-type target_name() :: atom().% Name associated to a config_key() though 'require' +-type key_or_name() :: config_key() | target_name(). + +%% Types used when logging connections with the 'cth_conn_log' hook +-type conn_log_options() :: [conn_log_option()]. +-type conn_log_option() :: {log_type,conn_log_type()} | + {hosts,[key_or_name()]}. +-type conn_log_type() :: raw | pretty | html | silent. +-type conn_log_mod() :: ct_netconfc | ct_telnet. +%%---------------------------------------------------------------------- + + + %%%----------------------------------------------------------------- %%% @spec install(Opts) -> ok | {error,Reason} %%% Opts = [Opt] @@ -592,7 +622,7 @@ log(X1,X2,X3,X4) -> %%% Format = string() %%% Args = list() %%% Opts = [Opt] -%%% Opt = esc_chars | no_css +%%% Opt = {heading,string()} | esc_chars | no_css %%% %%% @doc Printout from a test case to the log file. %%% @@ -610,43 +640,61 @@ log(Category,Importance,Format,Args,Opts) -> %%%----------------------------------------------------------------- %%% @spec print(Format) -> ok -%%% @equiv print(default,50,Format,[]) +%%% @equiv print(default,50,Format,[],[]) print(Format) -> - print(default,?STD_IMPORTANCE,Format,[]). + print(default,?STD_IMPORTANCE,Format,[],[]). %%%----------------------------------------------------------------- %%% @spec print(X1,X2) -> ok %%% X1 = Category | Importance | Format %%% X2 = Format | Args -%%% @equiv print(Category,Importance,Format,Args) +%%% @equiv print(Category,Importance,Format,Args,[]) print(X1,X2) -> {Category,Importance,Format,Args} = if is_atom(X1) -> {X1,?STD_IMPORTANCE,X2,[]}; is_integer(X1) -> {default,X1,X2,[]}; is_list(X1) -> {default,?STD_IMPORTANCE,X1,X2} end, - print(Category,Importance,Format,Args). + print(Category,Importance,Format,Args,[]). %%%----------------------------------------------------------------- %%% @spec print(X1,X2,X3) -> ok +%%% X1 = Category | Importance | Format +%%% X2 = Importance | Format | Args +%%% X3 = Format | Args | Opts +%%% @equiv print(Category,Importance,Format,Args,Opts) +print(X1,X2,X3) -> + {Category,Importance,Format,Args,Opts} = + if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[],[]}; + is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3,[]}; + is_integer(X1) -> {default,X1,X2,X3,[]}; + is_list(X1), is_list(X2) -> {default,?STD_IMPORTANCE,X1,X2,X3} + end, + print(Category,Importance,Format,Args,Opts). + +%%%----------------------------------------------------------------- +%%% @spec print(X1,X2,X3,X4) -> ok %%% X1 = Category | Importance %%% X2 = Importance | Format %%% X3 = Format | Args -%%% @equiv print(Category,Importance,Format,Args) -print(X1,X2,X3) -> - {Category,Importance,Format,Args} = - if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[]}; - is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3}; - is_integer(X1) -> {default,X1,X2,X3} +%%% X4 = Args | Opts +%%% @equiv print(Category,Importance,Format,Args,Opts) +print(X1,X2,X3,X4) -> + {Category,Importance,Format,Args,Opts} = + if is_atom(X1), is_integer(X2) -> {X1,X2,X3,X4,[]}; + is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3,X4}; + is_integer(X1) -> {default,X1,X2,X3,X4} end, - print(Category,Importance,Format,Args). + print(Category,Importance,Format,Args,Opts). %%%----------------------------------------------------------------- -%%% @spec print(Category,Importance,Format,Args) -> ok +%%% @spec print(Category,Importance,Format,Args,Opts) -> ok %%% Category = atom() %%% Importance = integer() %%% Format = string() %%% Args = list() +%%% Opts = [Opt] +%%% Opt = {heading,string()} %%% %%% @doc Printout from a test case to the console. %%% @@ -658,13 +706,13 @@ print(X1,X2,X3) -> %%% and default value for <c>Args</c> is <c>[]</c>.</p> %%% <p>Please see the User's Guide for details on <c>Category</c> %%% and <c>Importance</c>.</p> -print(Category,Importance,Format,Args) -> - ct_logs:tc_print(Category,Importance,Format,Args). +print(Category,Importance,Format,Args,Opts) -> + ct_logs:tc_print(Category,Importance,Format,Args,Opts). %%%----------------------------------------------------------------- %%% @spec pal(Format) -> ok -%%% @equiv pal(default,50,Format,[]) +%%% @equiv pal(default,50,Format,[],[]) pal(Format) -> pal(default,?STD_IMPORTANCE,Format,[]). @@ -672,35 +720,53 @@ pal(Format) -> %%% @spec pal(X1,X2) -> ok %%% X1 = Category | Importance | Format %%% X2 = Format | Args -%%% @equiv pal(Category,Importance,Format,Args) +%%% @equiv pal(Category,Importance,Format,Args,[]) pal(X1,X2) -> {Category,Importance,Format,Args} = if is_atom(X1) -> {X1,?STD_IMPORTANCE,X2,[]}; is_integer(X1) -> {default,X1,X2,[]}; is_list(X1) -> {default,?STD_IMPORTANCE,X1,X2} end, - pal(Category,Importance,Format,Args). + pal(Category,Importance,Format,Args,[]). %%%----------------------------------------------------------------- %%% @spec pal(X1,X2,X3) -> ok +%%% X1 = Category | Importance | Format +%%% X2 = Importance | Format | Args +%%% X3 = Format | Args | Opts +%%% @equiv pal(Category,Importance,Format,Args,Opts) +pal(X1,X2,X3) -> + {Category,Importance,Format,Args,Opts} = + if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[],[]}; + is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3,[]}; + is_integer(X1) -> {default,X1,X2,X3,[]}; + is_list(X1), is_list(X2) -> {default,?STD_IMPORTANCE,X1,X2,X3} + end, + pal(Category,Importance,Format,Args,Opts). + +%%%----------------------------------------------------------------- +%%% @spec pal(X1,X2,X3,X4) -> ok %%% X1 = Category | Importance %%% X2 = Importance | Format %%% X3 = Format | Args -%%% @equiv pal(Category,Importance,Format,Args) -pal(X1,X2,X3) -> - {Category,Importance,Format,Args} = - if is_atom(X1), is_integer(X2) -> {X1,X2,X3,[]}; - is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3}; - is_integer(X1) -> {default,X1,X2,X3} +%%% X4 = Args | Opts +%%% @equiv pal(Category,Importance,Format,Args,Opts) +pal(X1,X2,X3,X4) -> + {Category,Importance,Format,Args,Opts} = + if is_atom(X1), is_integer(X2) -> {X1,X2,X3,X4,[]}; + is_atom(X1), is_list(X2) -> {X1,?STD_IMPORTANCE,X2,X3,X4}; + is_integer(X1) -> {default,X1,X2,X3,X4} end, - pal(Category,Importance,Format,Args). + pal(Category,Importance,Format,Args,Opts). %%%----------------------------------------------------------------- -%%% @spec pal(Category,Importance,Format,Args) -> ok +%%% @spec pal(Category,Importance,Format,Args,Opts) -> ok %%% Category = atom() %%% Importance = integer() %%% Format = string() %%% Args = list() +%%% Opts = [Opt] +%%% Opt = {heading,string()} | no_css %%% %%% @doc Print and log from a test case. %%% @@ -712,8 +778,8 @@ pal(X1,X2,X3) -> %%% and default value for <c>Args</c> is <c>[]</c>.</p> %%% <p>Please see the User's Guide for details on <c>Category</c> %%% and <c>Importance</c>.</p> -pal(Category,Importance,Format,Args) -> - ct_logs:tc_pal(Category,Importance,Format,Args). +pal(Category,Importance,Format,Args,Opts) -> + ct_logs:tc_pal(Category,Importance,Format,Args,Opts). %%%----------------------------------------------------------------- %%% @spec set_verbosity(Category, Level) -> ok diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index 93e64c65fe..6c1e46925f 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -238,6 +238,8 @@ actionstr(#conn_log{action=cmd}) -> "----->"; actionstr(#conn_log{action=recv}) -> "<-----"; actionstr(#conn_log{action=open}) -> "opened session to"; actionstr(#conn_log{action=close}) -> "closed session to"; +actionstr(#conn_log{action=connect}) -> "connected to"; +actionstr(#conn_log{action=disconnect}) -> "disconnected from"; actionstr(_) -> "<---->". serverstr(#conn_log{name=undefined,address={undefined,_}}) -> diff --git a/lib/common_test/src/ct_default_gl.erl b/lib/common_test/src/ct_default_gl.erl new file mode 100644 index 0000000000..d1b52e5f4f --- /dev/null +++ b/lib/common_test/src/ct_default_gl.erl @@ -0,0 +1,83 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_default_gl). +-export([start_link/1, stop/0]). + +-export([init/1,handle_call/3,handle_cast/2,handle_info/2,terminate/2]). + +%% start_link() +%% Start a new group leader process. +start_link(ParentGL) -> + do_start(ParentGL, 3). + +do_start(_ParentGL, 0) -> + exit({?MODULE,startup}); +do_start(ParentGL, Retries) -> + case whereis(?MODULE) of + undefined -> + case gen_server:start_link(?MODULE, [ParentGL], []) of + {ok,Pid} -> + {ok,Pid}; + Other -> + Other + end; + Pid -> + exit(Pid, kill), + timer:sleep(1000), + do_start(ParentGL, Retries-1) + end. + +%% stop(Pid) +%% Stop a group leader process. +stop() -> + gen_server:cast(whereis(?MODULE), stop). + + +%%% Internal functions. + +init([ParentGL]) -> + register(?MODULE, self()), + {ok,#{parent_gl_pid => ParentGL, + parent_gl_monitor => erlang:monitor(process,ParentGL)}}. + +handle_cast(stop, St) -> + {stop,normal,St}. + +%% If the parent group leader dies, fall back on using the local user process +handle_info({'DOWN',Ref,process,_,_Reason}, #{parent_gl_monitor := Ref} = St) -> + User = whereis(user), + {noreply,St#{parent_gl_pid => User, + parent_gl_monitor => erlang:monitor(process,User)}}; + +handle_info({io_request,_From,_ReplyAs,_Req} = IoReq, + #{parent_gl_pid := ParentGL} = St) -> + ParentGL ! IoReq, + {noreply,St}; + +handle_info(Msg, St) -> + io:format(user, "Common Test Group Leader process got: ~tp~n", [Msg]), + {noreply,St}. + +handle_call(_Req, _From, St) -> + {reply,ok,St}. + +terminate(_, _) -> + ok. diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 104515e57e..141c7f5b0a 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -52,6 +52,10 @@ %%% %%% @doc Test server framework callback, called by the test_server %%% when a new test case is started. +init_tc(_,{end_per_testcase_not_run,_},[Config]) -> + %% Testcase is completed (skipped or failed), but end_per_testcase + %% is not run - don't call pre-hook. + {ok,[Config]}; init_tc(Mod,EPTC={end_per_testcase,_},[Config]) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Config), @@ -62,7 +66,7 @@ init_tc(Mod,EPTC={end_per_testcase,_},[Config]) -> Other end; -init_tc(Mod,Func0,Args) -> +init_tc(Mod,Func0,Args) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), {Func,HookFunc} = case Func0 of @@ -84,12 +88,15 @@ init_tc(Mod,Func0,Args) -> andalso Func=/=end_per_group andalso ct_util:get_testdata(skip_rest) of true -> + initialize(false,Mod,Func,Args), {auto_skip,"Repeated test stopped by force_stop option"}; _ -> case ct_util:get_testdata(curr_tc) of {Suite,{suite0_failed,{require,Reason}}} -> + initialize(false,Mod,Func,Args), {auto_skip,{require_failed_in_suite0,Reason}}; {Suite,{suite0_failed,_}=Failure} -> + initialize(false,Mod,Func,Args), {fail,Failure}; _ -> ct_util:update_testdata(curr_tc, @@ -118,16 +125,14 @@ init_tc(Mod,Func0,Args) -> end, init_tc1(Mod,Suite,Func,HookFunc,Args); {failed,Seq,BadFunc} -> - {auto_skip,{sequence_failed,Seq,BadFunc}} + initialize(false,Mod,Func,Args), + {auto_skip,{sequence_failed,Seq,BadFunc}} end end end. init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={?MODULE,error_in_suite}}), + initialize(false,?MODULE,error_in_suite), _ = ct_suite_init(?MODULE,error_in_suite,[],Config0), case ?val(error,Config0) of undefined -> @@ -177,27 +182,21 @@ init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) -> ct_config:delete_default_config(testcase), HookFunc end, - Initialize = fun() -> - ct_logs:init_tc(false), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}) - end, case add_defaults(Mod,Func,AllGroups) of Error = {suite0_failed,_} -> - Initialize(), + initialize(false,Mod,FuncSpec), ct_util:set_testdata({curr_tc,{Suite,Error}}), {error,Error}; Error = {group0_failed,_} -> - Initialize(), + initialize(false,Mod,FuncSpec), {auto_skip,Error}; Error = {testcase0_failed,_} -> - Initialize(), + initialize(false,Mod,FuncSpec), {auto_skip,Error}; {SuiteInfo,MergeResult} -> case MergeResult of {error,Reason} -> - Initialize(), + initialize(false,Mod,FuncSpec), {fail,Reason}; _ -> init_tc2(Mod,Suite,Func,HookFunc1, @@ -236,11 +235,8 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) -> Conns -> ct_util:silence_connections(Conns) end, - ct_logs:init_tc(Func == init_per_suite), FuncSpec = group_or_func(Func,Config), - ct_event:notify(#event{name=tc_start, - node=node(), - data={Mod,FuncSpec}}), + initialize((Func==init_per_suite),Mod,FuncSpec), case catch configure(MergedInfo,MergedInfo,SuiteInfo, FuncSpec,[],Config) of @@ -268,6 +264,18 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) -> end end. +initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) -> + initialize(RefreshLogs,Mod,group_or_func(Func,Config)); +initialize(RefreshLogs,Mod,Func,_) -> + initialize(RefreshLogs,Mod,Func). + +initialize(RefreshLogs,Mod,FuncSpec) -> + ct_logs:init_tc(RefreshLogs), + ct_event:notify(#event{name=tc_start, + node=node(), + data={Mod,FuncSpec}}). + + ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) -> case ct_hooks:init_tc(Suite,HookFunc,Config) of NewConfig when is_list(NewConfig) -> @@ -307,7 +315,7 @@ add_defaults(Mod,Func, GroupPath) -> "~w:suite/0 failed: ~p~n", [Suite,Reason]), io:format(ErrStr, []), - io:format(user, ErrStr, []), + io:format(?def_gl, ErrStr, []), {suite0_failed,{exited,Reason}}; SuiteInfo when is_list(SuiteInfo) -> case lists:all(fun(E) when is_tuple(E) -> true; @@ -330,7 +338,7 @@ add_defaults(Mod,Func, GroupPath) -> "~w:suite/0: ~p~n", [Suite,SuiteInfo]), io:format(ErrStr, []), - io:format(user, ErrStr, []), + io:format(?def_gl, ErrStr, []), {suite0_failed,bad_return_value} end; SuiteInfo -> @@ -338,7 +346,7 @@ add_defaults(Mod,Func, GroupPath) -> "Invalid return value from " "~w:suite/0: ~p~n", [Suite,SuiteInfo]), io:format(ErrStr, []), - io:format(user, ErrStr, []), + io:format(?def_gl, ErrStr, []), {suite0_failed,bad_return_value} end. @@ -366,7 +374,7 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) -> "~w:group(~w): ~p~n", [Mod,GrName,BadGr0Val]), io:format(Gr0ErrStr, []), - io:format(user, Gr0ErrStr, []), + io:format(?def_gl, Gr0ErrStr, []), {group0_failed,bad_return_value}; _ -> Args = if Func == init_per_group ; Func == end_per_group -> @@ -388,7 +396,7 @@ add_defaults1(Mod,Func, GroupPath, SuiteInfo) -> "~w:~w/0: ~p~n", [Mod,Func,BadTC0Val]), io:format(TC0ErrStr, []), - io:format(user, TC0ErrStr, []), + io:format(?def_gl, TC0ErrStr, []), {testcase0_failed,bad_return_value}; _ -> %% let test case info (also for all config funcs) override @@ -675,22 +683,35 @@ end_tc(Mod,Func,{Result,[Args]}, Return) -> end_tc(Mod,Func,self(),Result,Args,Return). end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> - %% in case Mod == ct_framework, lookup the suite name - Suite = get_suite_name(Mod, Args), - case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of - '$ct_no_change' -> - ok; - HookResult -> - HookResult + case end_hook_func(IPTC,Return,IPTC) of + undefined -> ok; + _ -> + %% in case Mod == ct_framework, lookup the suite name + Suite = get_suite_name(Mod, Args), + case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of + '$ct_no_change' -> + ok; + HookResult -> + HookResult + end end; end_tc(Mod,Func0,TCPid,Result,Args,Return) -> %% in case Mod == ct_framework, lookup the suite name Suite = get_suite_name(Mod, Args), - {EPTC,Func} = case Func0 of - {end_per_testcase,F} -> {true,F}; - _ -> {false,Func0} - end, + {Func,FuncSpec,HookFunc} = + case Func0 of + {end_per_testcase_not_run,F} -> + %% Testcase is completed (skipped or failed), but + %% end_per_testcase is not run - don't call post-hook. + {F,F,undefined}; + {end_per_testcase,F} -> + {F,F,Func0}; + _ -> + FS = group_or_func(Func0,Args), + HF = end_hook_func(Func0,Return,FS), + {Func0,FS,HF} + end, test_server:timetrap_cancel(), @@ -717,20 +738,18 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, ct_util:delete_suite_data(last_saved_config), - {FuncSpec,HookFunc} = - if not EPTC -> - FS = group_or_func(Func,Args), - {FS,FS}; - true -> - {Func,Func0} - end, {Result1,FinalNotify} = - case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of - '$ct_no_change' -> - {ok,Result}; - HookResult -> - {HookResult,HookResult} - end, + case HookFunc of + undefined -> + {ok,Result}; + _ -> + case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of + '$ct_no_change' -> + {ok,Result}; + HookResult -> + {HookResult,HookResult} + end + end, FinalResult = case get('$test_server_framework_test') of undefined -> @@ -821,6 +840,34 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) -> end, FinalResult. +%% This is to make sure that no post_init_per_* is ever called if the +%% corresponding pre_init_per_* was not called. +%% The skip or fail reasons are those that can be returned from +%% init_tc above in situations where we never came to call +%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call +%% ct_hooks:init_tc for init_per_suite, and thus we must not call +%% ct_hooks:end_tc for init_per_suite either. +end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) -> + undefined; +end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) -> + undefined; +end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) -> + undefined; +end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_) + when InfoFuncError==testcase0_failed; + InfoFuncError==require_failed -> + undefined; +end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_) + when InfoFuncError==group0_failed; + InfoFuncError==require_failed -> + undefined; +end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) -> + undefined; +end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) -> + undefined; +end_hook_func(_,_,Default) -> + Default. + %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} | %% {testcase_aborted,Reason} | testcase_aborted_or_killed | %% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} | @@ -927,7 +974,7 @@ error_notification(Mod,Func,_Args,{Error,Loc}) -> Div = "~n- - - - - - - - - - - - - - - - - - - " "- - - - - - - - - - - - - - - - - - - - -~n", ErrorStr2 = io_lib:format(ErrorFormat, ErrorArgs), - io:format(user, lists:concat([Div,ErrorStr2,Div,"~n"]), + io:format(?def_gl, lists:concat([Div,ErrorStr2,Div,"~n"]), []), Link = "\n\n<a href=\"#end\">" @@ -1133,7 +1180,7 @@ get_all(Mod, ConfTests) -> ErrStr = io_lib:format("~n*** ERROR *** " "~w:all/0 failed: ~p~n", [Mod,ExitReason]), - io:format(user, ErrStr, []), + io:format(?def_gl, ErrStr, []), %% save the error info so it doesn't get printed twice ct_util:set_testdata_async({{error_in_suite,Mod}, ExitReason}); @@ -1339,25 +1386,25 @@ report(What,Data) -> ok; tc_done -> {Suite,{Func,GrName},Result} = Data, - Data1 = if GrName == undefined -> {Suite,Func,Result}; - true -> Data - end, + FuncSpec = if GrName == undefined -> Func; + true -> {Func,GrName} + end, %% Register the group leader for the process calling the report %% function, making it possible for a hook function to print %% in the test case log file ReportingPid = self(), ct_logs:register_groupleader(ReportingPid, group_leader()), case Result of - {failed, _} -> - ct_hooks:on_tc_fail(What, Data1); - {skipped,{failed,{_,init_per_testcase,_}}} -> - ct_hooks:on_tc_skip(tc_auto_skip, Data1); - {skipped,{require_failed,_}} -> - ct_hooks:on_tc_skip(tc_auto_skip, Data1); - {skipped,_} -> - ct_hooks:on_tc_skip(tc_user_skip, Data1); - {auto_skipped,_} -> - ct_hooks:on_tc_skip(tc_auto_skip, Data1); + {failed, Reason} -> + ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason}); + {skipped,{failed,{_,init_per_testcase,_}}=Reason} -> + ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); + {skipped,{require_failed,_}=Reason} -> + ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); + {skipped,Reason} -> + ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason}); + {auto_skipped,Reason} -> + ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); _Else -> ok end, diff --git a/lib/common_test/src/ct_gen_conn.erl b/lib/common_test/src/ct_gen_conn.erl index 8b59d3ab23..88b05cf955 100644 --- a/lib/common_test/src/ct_gen_conn.erl +++ b/lib/common_test/src/ct_gen_conn.erl @@ -29,13 +29,6 @@ -export([call/2, call/3, return/2, do_within_time/2]). -export([log/3, start_log/1, cont_log/2, cont_log_no_timestamp/2, end_log/0]). -%%---------------------------------------------------------------------- -%% Exported types -%%---------------------------------------------------------------------- --export_type([server_id/0, - target_name/0, - key_or_name/0]). - -ifdef(debug). -define(dbg,true). -else. @@ -54,18 +47,6 @@ cb_state, ct_util_server}). -%%------------------------------------------------------------------ -%% Type declarations -%%------------------------------------------------------------------ --type server_id() :: atom(). -%% A `ServerId' which exists in a configuration file. --type target_name() :: atom(). -%% A name which is associated to a `server_id()' via a -%% `require' statement or a call to {@link ct:require/2} in the -%% test suite. --type key_or_name() :: server_id() | target_name(). - - %%%----------------------------------------------------------------- %%% @spec start(Address,InitData,CallbackMod,Opts) -> %%% {ok,Handle} | {error,Reason} @@ -373,8 +354,9 @@ loop(Opts) -> end; {stop, From} -> ct_util:unregister_connection(self()), - (Opts#gen_opts.callback):terminate(Opts#gen_opts.conn_pid, - Opts#gen_opts.cb_state), + ConnPid = Opts#gen_opts.conn_pid, + unlink(ConnPid), + (Opts#gen_opts.callback):terminate(ConnPid,Opts#gen_opts.cb_state), return(From,ok), ok; {{retry,{Error,_Name,CPid,_Msg}}, From} when @@ -411,8 +393,9 @@ loop(Opts) -> loop(Opts#gen_opts{cb_state=NewState}); {stop,Reply,NewState} -> ct_util:unregister_connection(self()), - (Opts#gen_opts.callback):terminate(Opts#gen_opts.conn_pid, - NewState), + ConnPid = Opts#gen_opts.conn_pid, + unlink(ConnPid), + (Opts#gen_opts.callback):terminate(ConnPid,NewState), return(From,Reply) end; Msg when Opts#gen_opts.forward==true -> @@ -422,8 +405,9 @@ loop(Opts) -> loop(Opts#gen_opts{cb_state=NewState}); {stop,NewState} -> ct_util:unregister_connection(self()), - (Opts#gen_opts.callback):terminate(Opts#gen_opts.conn_pid, - NewState) + ConnPid = Opts#gen_opts.conn_pid, + unlink(ConnPid), + (Opts#gen_opts.callback):terminate(ConnPid,NewState) end end. diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl index 1375e7dcc7..333151ee1b 100644 --- a/lib/common_test/src/ct_groups.erl +++ b/lib/common_test/src/ct_groups.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -442,17 +442,21 @@ make_conf(Mod, Name, Props, TestSpec) -> ok end, {InitConf,EndConf,ExtraProps} = - case erlang:function_exported(Mod,init_per_group,2) of - true -> - {{Mod,init_per_group},{Mod,end_per_group},[]}; - false -> + case {erlang:function_exported(Mod,init_per_group,2), + erlang:function_exported(Mod,end_per_group,2)} of + {false,false} -> ct_logs:log("TEST INFO", "init_per_group/2 and " "end_per_group/2 missing for group " "~w in ~w, using default.", [Name,Mod]), {{ct_framework,init_per_group}, {ct_framework,end_per_group}, - [{suite,Mod}]} + [{suite,Mod}]}; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + {{Mod,init_per_group},{Mod,end_per_group},[]} end, {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index c9a4abb5ee..8cdc6d8c75 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -92,15 +92,17 @@ init_tc(Mod, end_per_suite, Config) -> call(fun call_generic/3, Config, [pre_end_per_suite, Mod]); init_tc(Mod, {init_per_group, GroupName, Properties}, Config) -> maybe_start_locker(Mod, GroupName, Properties), - call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); -init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> - call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); -init_tc(_Mod, {init_per_testcase,TC}, Config) -> - call(fun call_generic/3, Config, [pre_init_per_testcase, TC]); -init_tc(_Mod, {end_per_testcase,TC}, Config) -> - call(fun call_generic/3, Config, [pre_end_per_testcase, TC]); -init_tc(_Mod, TC = error_in_suite, Config) -> - call(fun call_generic/3, Config, [pre_init_per_testcase, TC]). + call(fun call_generic_fallback/3, Config, + [pre_init_per_group, Mod, GroupName]); +init_tc(Mod, {end_per_group, GroupName, _}, Config) -> + call(fun call_generic_fallback/3, Config, + [pre_end_per_group, Mod, GroupName]); +init_tc(Mod, {init_per_testcase,TC}, Config) -> + call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]); +init_tc(Mod, {end_per_testcase,TC}, Config) -> + call(fun call_generic_fallback/3, Config, [pre_end_per_testcase, Mod, TC]); +init_tc(Mod, TC = error_in_suite, Config) -> + call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]). %% @doc Called as each test case is completed. This includes all configuration %% tests. @@ -126,23 +128,23 @@ end_tc(Mod, init_per_suite, Config, _Result, Return) -> end_tc(Mod, end_per_suite, Config, Result, _Return) -> call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config], '$ct_no_change'); -end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> - call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], - '$ct_no_change'); +end_tc(Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> + call(fun call_generic_fallback/3, Return, + [post_init_per_group, Mod, GroupName, Config], '$ct_no_change'); end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) -> - Res = call(fun call_generic/3, Result, - [post_end_per_group, GroupName, Config], '$ct_no_change'), + Res = call(fun call_generic_fallback/3, Result, + [post_end_per_group, Mod, GroupName, Config], '$ct_no_change'), maybe_stop_locker(Mod, GroupName, Properties), Res; -end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) -> - call(fun call_generic/3, Result, [post_init_per_testcase, TC, Config], - '$ct_no_change'); -end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) -> - call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], - '$ct_no_change'); -end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) -> - call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], - '$ct_no_change'). +end_tc(Mod, {init_per_testcase,TC}, Config, Result, _Return) -> + call(fun call_generic_fallback/3, Result, + [post_init_per_testcase, Mod, TC, Config], '$ct_no_change'); +end_tc(Mod, {end_per_testcase,TC}, Config, Result, _Return) -> + call(fun call_generic_fallback/3, Result, + [post_end_per_testcase, Mod, TC, Config], '$ct_no_change'); +end_tc(Mod, TC = error_in_suite, Config, Result, _Return) -> + call(fun call_generic_fallback/3, Result, + [post_end_per_testcase, Mod, TC, Config], '$ct_no_change'). %% Case = TestCase | {TestCase,GroupName} @@ -181,15 +183,21 @@ call_terminate(#ct_hook_config{ module = Mod, state = State} = Hook, _, _) -> {[],Hook}. call_cleanup(#ct_hook_config{ module = Mod, state = State} = Hook, - Reason, [Function, _Suite | Args]) -> + Reason, [Function | Args]) -> NewState = catch_apply(Mod,Function, Args ++ [Reason, State], - State), + State, true), {Reason, Hook#ct_hook_config{ state = NewState } }. -call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, - Value, [Function | Args]) -> +call_generic(Hook, Value, Meta) -> + do_call_generic(Hook, Value, Meta, false). + +call_generic_fallback(Hook, Value, Meta) -> + do_call_generic(Hook, Value, Meta, true). + +do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, + Value, [Function | Args], Fallback) -> {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], - {Value,State}), + {Value,State}, Fallback), {NewValue, Hook#ct_hook_config{ state = NewState } }. %% Generic call function @@ -257,15 +265,15 @@ remove(Key,List) when is_list(List) -> remove(_, Else) -> Else. -%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc -scope([pre_init_per_testcase, TC|_]) -> - [post_init_per_testcase, TC]; -scope([pre_end_per_testcase, TC|_]) -> - [post_end_per_testcase, TC]; -scope([pre_init_per_group, GroupName|_]) -> - [post_end_per_group, GroupName]; -scope([post_init_per_group, GroupName|_]) -> - [post_end_per_group, GroupName]; +%% Translate scopes, i.e. is_tuplenit_per_group,group1 -> end_per_group,group1 etc +scope([pre_init_per_testcase, SuiteName, TC|_]) -> + [post_init_per_testcase, SuiteName, TC]; +scope([pre_end_per_testcase, SuiteName, TC|_]) -> + [post_end_per_testcase, SuiteName, TC]; +scope([pre_init_per_group, SuiteName, GroupName|_]) -> + [post_end_per_group, SuiteName, GroupName]; +scope([post_init_per_group, SuiteName, GroupName|_]) -> + [post_end_per_group, SuiteName, GroupName]; scope([pre_init_per_suite, SuiteName|_]) -> [post_end_per_suite, SuiteName]; scope([post_init_per_suite, SuiteName|_]) -> @@ -273,14 +281,29 @@ scope([post_init_per_suite, SuiteName|_]) -> scope(init) -> none. -terminate_if_scope_ends(HookId, [on_tc_skip,_Suite,{end_per_group,Name}], +strip_config([post_init_per_testcase, SuiteName, TC|_]) -> + [post_init_per_testcase, SuiteName, TC]; +strip_config([post_end_per_testcase, SuiteName, TC|_]) -> + [post_end_per_testcase, SuiteName, TC]; +strip_config([post_init_per_group, SuiteName, GroupName|_]) -> + [post_init_per_group, SuiteName, GroupName]; +strip_config([post_end_per_group, SuiteName, GroupName|_]) -> + [post_end_per_group, SuiteName, GroupName]; +strip_config([post_init_per_suite, SuiteName|_]) -> + [post_init_per_suite, SuiteName]; +strip_config([post_end_per_suite, SuiteName|_]) -> + [post_end_per_suite, SuiteName]; +strip_config(Other) -> + Other. + + +terminate_if_scope_ends(HookId, [on_tc_skip,Suite,{end_per_group,Name}], Hooks) -> - terminate_if_scope_ends(HookId, [post_end_per_group, Name], Hooks); + terminate_if_scope_ends(HookId, [post_end_per_group, Suite, Name], Hooks); terminate_if_scope_ends(HookId, [on_tc_skip,Suite,end_per_suite], Hooks) -> terminate_if_scope_ends(HookId, [post_end_per_suite, Suite], Hooks); -terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] -> - terminate_if_scope_ends(HookId,[Function,Tag],Hooks); -terminate_if_scope_ends(HookId, Function, Hooks) -> +terminate_if_scope_ends(HookId, Function0, Hooks) -> + Function = strip_config(Function0), case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of #ct_hook_config{ id = HookId, scope = Function} = Hook -> terminate([Hook]), @@ -384,21 +407,29 @@ pos(Id,[_|Rest],Num) -> catch_apply(M,F,A, Default) -> + catch_apply(M,F,A,Default,false). +catch_apply(M,F,A, Default, Fallback) -> + not erlang:module_loaded(M) andalso (catch M:module_info()), + case erlang:function_exported(M,F,length(A)) of + false when Fallback -> + catch_apply(M,F,tl(A),Default,false); + false -> + Default; + true -> + catch_apply(M,F,A) + end. + +catch_apply(M,F,A) -> try - erlang:apply(M,F,A) + erlang:apply(M,F,A) catch _:Reason -> - case erlang:get_stacktrace() of - %% Return the default if it was the CTH module which did not have the function. - [{M,F,A,_}|_] when Reason == undef -> - Default; - Trace -> - ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p", - [error,{Reason,Trace}]), - throw({error_in_cth_call, - lists:flatten( - io_lib:format("~w:~w/~w CTH call failed", - [M,F,length(A)]))}) - end + Trace = erlang:get_stacktrace(), + ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p", + [error,{Reason,Trace}]), + throw({error_in_cth_call, + lists:flatten( + io_lib:format("~w:~w/~w CTH call failed", + [M,F,length(A)]))}) end. diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 9282a9f81d..978af0f149 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2016. All Rights Reserved. +%% Copyright Ericsson AB 2003-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -41,12 +41,13 @@ -export([xhtml/2, locate_priv_file/1, make_relative/1]). -export([insert_javascript/1]). -export([uri/1]). +-export([parse_keep_logs/1]). %% Logging stuff directly from testcase -export([tc_log/3, tc_log/4, tc_log/5, tc_log/6, tc_log_async/3, tc_log_async/5, - tc_print/3, tc_print/4, - tc_pal/3, tc_pal/4, ct_log/3, + tc_print/3, tc_print/4, tc_print/5, + tc_pal/3, tc_pal/4, tc_pal/5, ct_log/3, basic_html/0]). %% Simulate logger process for use without ct environment running @@ -447,10 +448,10 @@ tc_log(Category,Importance,Format,Args,Opts) -> tc_log(Category,Importance,"User",Format,Args,Opts). %%%----------------------------------------------------------------- -%%% @spec tc_log(Category,Importance,Printer,Format,Args,Opts) -> ok +%%% @spec tc_log(Category,Importance,Heading,Format,Args,Opts) -> ok %%% Category = atom() %%% Importance = integer() -%%% Printer = string() +%%% Heading = string() %%% Format = string() %%% Args = list() %%% Opts = list() @@ -460,13 +461,18 @@ tc_log(Category,Importance,Format,Args,Opts) -> %%% <p>This function is called by <code>ct</code> when logging %%% stuff directly from a testcase (i.e. not from within the CT %%% framework).</p> -tc_log(Category,Importance,Printer,Format,Args,Opts) -> +tc_log(Category,Importance,Heading,Format,Args,Opts) -> Data = case lists:member(no_css, Opts) of true -> [{Format,Args}]; false -> - [{hd,div_header(Category,Printer),[]}, + Heading1 = + case proplists:get_value(heading, Opts) of + undefined -> Heading; + Str -> Str + end, + [{hd,div_header(Category,Heading1),[]}, {Format,Args}, {ft,div_footer(),[]}] end, @@ -484,7 +490,7 @@ tc_log_async(Category,Format,Args) -> %%% @spec tc_log_async(Category,Importance,Format,Args) -> ok %%% Category = atom() %%% Importance = integer() -%%% Printer = string() +%%% Heading = string() %%% Format = string() %%% Args = list() %%% @@ -495,31 +501,38 @@ tc_log_async(Category,Format,Args) -> %%% to avoid deadlocks when e.g. the hook that handles SASL printouts %%% prints to the test case log file at the same time test server %%% asks ct_logs for an html wrapper.</p> -tc_log_async(Category,Importance,Printer,Format,Args) -> +tc_log_async(Category,Importance,Heading,Format,Args) -> cast({log,async,self(),group_leader(),Category,Importance, - [{hd,div_header(Category,Printer),[]}, + [{hd,div_header(Category,Heading),[]}, {Format,Args}, {ft,div_footer(),[]}], true}), ok. %%%----------------------------------------------------------------- %%% @spec tc_print(Category,Format,Args) -%%% @equiv tc_print(Category,?STD_IMPORTANCE,Format,Args) +%%% @equiv tc_print(Category,?STD_IMPORTANCE,Format,Args,[]) tc_print(Category,Format,Args) -> - tc_print(Category,?STD_IMPORTANCE,Format,Args). + tc_print(Category,?STD_IMPORTANCE,Format,Args,[]). + +%%%----------------------------------------------------------------- +%%% @spec tc_print(Category,Importance,Format,Args) +%%% @equiv tc_print(Category,Importance,Format,Args,[]) +tc_print(Category,Importance,Format,Args) -> + tc_print(Category,Importance,Format,Args,[]). %%%----------------------------------------------------------------- -%%% @spec tc_print(Category,Importance,Format,Args) -> ok +%%% @spec tc_print(Category,Importance,Format,Args,Opts) -> ok %%% Category = atom() %%% Importance = integer() %%% Format = string() %%% Args = list() +%%% Opts = list() %%% %%% @doc Console printout from a testcase. %%% %%% <p>This function is called by <code>ct</code> when printing %%% stuff from a testcase on the user console.</p> -tc_print(Category,Importance,Format,Args) -> +tc_print(Category,Importance,Format,Args,Opts) -> VLvl = case ct_util:get_verbosity(Category) of undefined -> ct_util:get_verbosity('$unspecified'); @@ -531,50 +544,61 @@ tc_print(Category,Importance,Format,Args) -> Val end, if Importance >= (100-VLvl) -> - Head = get_heading(Category), - io:format(user, lists:concat([Head,Format,"\n\n"]), Args), + Heading = + case proplists:get_value(heading, Opts) of + undefined -> atom_to_list(Category); + Hd -> Hd + end, + Str = lists:concat([get_header(Heading),Format,"\n\n"]), + try + io:format(?def_gl, Str, Args) + catch + %% default group leader probably not started, or has stopped + _:_ -> io:format(user, Str, Args) + end, ok; true -> ok end. -get_heading(default) -> +get_header("default") -> io_lib:format("\n-----------------------------" "-----------------------\n~s\n", [log_timestamp(?now)]); -get_heading(Category) -> +get_header(Heading) -> io_lib:format("\n-----------------------------" - "-----------------------\n~s ~w\n", - [log_timestamp(?now),Category]). + "-----------------------\n~s ~s\n", + [Heading,log_timestamp(?now)]). %%%----------------------------------------------------------------- %%% @spec tc_pal(Category,Format,Args) -> ok -%%% @equiv tc_pal(Category,?STD_IMPORTANCE,Format,Args) -> ok +%%% @equiv tc_pal(Category,?STD_IMPORTANCE,Format,Args,[]) -> ok tc_pal(Category,Format,Args) -> - tc_pal(Category,?STD_IMPORTANCE,Format,Args). + tc_pal(Category,?STD_IMPORTANCE,Format,Args,[]). %%%----------------------------------------------------------------- %%% @spec tc_pal(Category,Importance,Format,Args) -> ok +%%% @equiv tc_pal(Category,Importance,Format,Args,[]) -> ok +tc_pal(Category,Importance,Format,Args) -> + tc_pal(Category,Importance,Format,Args,[]). + +%%%----------------------------------------------------------------- +%%% @spec tc_pal(Category,Importance,Format,Args,Opts) -> ok %%% Category = atom() %%% Importance = integer() %%% Format = string() %%% Args = list() +%%% Opts = list() %%% %%% @doc Print and log from a testcase. %%% %%% <p>This function is called by <code>ct</code> when logging %%% stuff directly from a testcase. The info is written both in the %%% log and on the console.</p> -tc_pal(Category,Importance,Format,Args) -> - tc_print(Category,Importance,Format,Args), - cast({log,sync,self(),group_leader(),Category,Importance, - [{hd,div_header(Category),[]}, - {Format,Args}, - {ft,div_footer(),[]}], - true}), - ok. - +tc_pal(Category,Importance,Format,Args,Opts) -> + tc_print(Category,Importance,Format,Args,Opts), + tc_log(Category,Importance,"User",Format,Args,[esc_chars|Opts]). %%%----------------------------------------------------------------- %%% @spec ct_log(Category,Format,Args) -> ok @@ -603,9 +627,9 @@ int_footer() -> div_header(Class) -> div_header(Class,"User"). -div_header(Class,Printer) -> +div_header(Class,Heading) -> "\n</pre>\n<div class=\"" ++ atom_to_list(Class) ++ "\"><pre><b>*** " - ++ Printer ++ " " ++ log_timestamp(?now) ++ " ***</b>". + ++ Heading ++ " " ++ log_timestamp(?now) ++ " ***</b>". div_footer() -> "</pre></div>\n<pre>". @@ -679,7 +703,7 @@ logger(Parent, Mode, Verbosity) -> PrivFilesDestRun = [filename:join(AbsDir, F) || F <- PrivFiles], case copy_priv_files(PrivFilesSrc, PrivFilesDestTop) of {error,Src1,Dest1,Reason1} -> - io:format(user, "ERROR! "++ + io:format(?def_gl, "ERROR! "++ "Priv file ~p could not be copied to ~p. "++ "Reason: ~p~n", [Src1,Dest1,Reason1]), @@ -687,7 +711,7 @@ logger(Parent, Mode, Verbosity) -> ok -> case copy_priv_files(PrivFilesSrc, PrivFilesDestRun) of {error,Src2,Dest2,Reason2} -> - io:format(user, + io:format(?def_gl, "ERROR! "++ "Priv file ~p could not be copied to ~p. " ++"Reason: ~p~n", @@ -1923,7 +1947,11 @@ make_all_runs_index(When) -> end, Dirs = filelib:wildcard(logdir_prefix()++"*.*"), - DirsSorted = (catch sort_all_runs(Dirs)), + DirsSorted0 = (catch sort_all_runs(Dirs)), + DirsSorted = + if When == start -> DirsSorted0; + true -> maybe_delete_old_dirs(DirsSorted0) + end, LogCacheInfo = get_cache_data(UseCache), @@ -2041,6 +2069,36 @@ sort_ct_runs(Dirs) -> {DateHH1,MM1,SS1} =< {DateHH2,MM2,SS2} end, Dirs). +parse_keep_logs([Str="all"]) -> + parse_keep_logs(list_to_atom(Str)); +parse_keep_logs([NStr]) -> + parse_keep_logs(list_to_integer(NStr)); +parse_keep_logs(all) -> + all; +parse_keep_logs(N) when is_integer(N), N>0 -> + N. + +maybe_delete_old_dirs(Sorted) -> + {Keep,Delete} = + case application:get_env(common_test, keep_logs) of + {ok,MaxN} when is_integer(MaxN), length(Sorted)>MaxN -> + lists:split(MaxN,Sorted); + _ -> + {Sorted,[]} + end, + delete_old_dirs(Delete), + Keep. + +delete_old_dirs([]) -> + ok; +delete_old_dirs(Dirs) -> + io:put_chars("\n Removing old test directories:\n"), + [begin + io:put_chars(" " ++ Dir ++ "\n"), + rm_dir(Dir) + end|| Dir <- Dirs], + ok. + dir_diff_all_runs(Dirs, LogCache) -> case LogCache#log_cache.all_runs of [] -> diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index ff45407fe0..b77570c121 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -23,24 +23,15 @@ %% Description: %% This file contains the Netconf client interface %% -%% @author Support +%% Netconf servers can be configured by adding the following statement +%% to a configuration file: %% -%% @doc Netconf client module. +%% {server_id(),options()}. %% -%% <p>The Netconf client is compliant with RFC4741 and RFC4742.</p> +%% The server_id() or an associated ct:target_name() shall then be +%% used in calls to open/2 connect/2. %% -%% <p> For each server to test against, the following entry can be -%% added to a configuration file:</p> -%% -%% <p>`{server_id(),options()}.'</p> -%% -%% <p> The `server_id()' or an associated `target_name()' (see -%% {@link ct}) shall then be used in calls to {@link open/2}.</p> -%% -%% <p>If no configuration exists for a server, a session can still be -%% opened by calling {@link open/2} with all necessary options given -%% in the call. The first argument to {@link open/2} can then be any -%% atom.</p> +%% If no configuration exists for a server, use open/1 and connect/1. %% %% == Logging == %% @@ -49,102 +40,15 @@ %% `ct_conn_log_h'. To use this error handler, add the `cth_conn_log' %% hook in your test suite, e.g. %% -%% ``` %% suite() -> -%% [{ct_hooks, [{cth_conn_log, [{conn_mod(),hook_options()}]}]}]. -%%''' -%% -%% The `conn_mod()' is the name of the common_test module implementing -%% the connection protocol, e.g. `ct_netconfc'. -%% -%% The hook option `log_type' specifies the type of logging: -%% -%% <dl> -%% <dt>`raw'</dt> -%% <dd>The sent and received netconf data is logged to a separate -%% text file as is without any formatting. A link to the file is -%% added to the test case HTML log.</dd> -%% -%% <dt>`pretty'</dt> -%% <dd>The sent and received netconf data is logged to a separate -%% text file with XML data nicely indented. A link to the file is -%% added to the test case HTML log.</dd> -%% -%% <dt>`html (default)'</dt> -%% <dd>The sent and received netconf traffic is pretty printed -%% directly in the test case HTML log.</dd> -%% -%% <dt>`silent'</dt> -%% <dd>Netconf traffic is not logged.</dd> -%% </dl> -%% -%% By default, all netconf traffic is logged in one single log -%% file. However, it is possible to have different connections logged -%% in separate files. To do this, use the hook option `hosts' and -%% list the names of the servers/connections that will be used in the -%% suite. Note that the connections must be named for this to work, -%% i.e. they must be opened with {@link open/2}. -%% -%% The `hosts' option has no effect if `log_type' is set to `html' or -%% `silent'. -%% -%% The hook options can also be specified in a configuration file with -%% the configuration variable `ct_conn_log': -%% -%% ``` -%% {ct_conn_log,[{conn_mod(),hook_options()}]}. -%% ''' +%% [{ct_hooks, [{cth_conn_log, [{ct:conn_log_mod(),ct:conn_log_options()}]}]}]. %% %% For example: %% -%% ``` -%% {ct_conn_log,[{ct_netconfc,[{log_type,pretty}, -%% {hosts,[key_or_name()]}]}]} -%% ''' -%% -%% <b>Note</b> that hook options specified in a configuration file -%% will overwrite the hardcoded hook options in the test suite. -%% -%% === Logging example 1 === -%% -%% The following `ct_hooks' statement will cause pretty printing of -%% netconf traffic to separate logs for the connections named -%% `nc_server1' and `nc_server2'. Any other connections will be logged -%% to default netconf log. -%% -%% ``` %% suite() -> -%% [{ct_hooks, [{cth_conn_log, [{ct_netconfc,[{log_type,pretty}}, -%% {hosts,[nc_server1,nc_server2]}]} -%% ]}]}]. -%%''' -%% -%% Connections must be opened like this: -%% -%% ``` -%% open(nc_server1,[...]), -%% open(nc_server2,[...]). -%% ''' -%% -%% === Logging example 2 === -%% -%% The following configuration file will cause raw logging of all -%% netconf traffic into one single text file. -%% -%% ``` -%% {ct_conn_log,[{ct_netconfc,[{log_type,raw}]}]}. -%% ''' -%% -%% The `ct_hooks' statement must look like this: -%% -%% ``` -%% suite() -> -%% [{ct_hooks, [{cth_conn_log, []}]}]. -%% ''' -%% -%% The same `ct_hooks' statement without the configuration file would -%% cause HTML logging of all netconf connections into the test case -%% HTML log. +%% [{ct_hooks, +%% [{cth_conn_log,[{ct_netconfc,[{log_type,pretty}, +%% {hosts,[my_configured_server]}]}]} %% %% == Notifications == %% @@ -152,11 +56,9 @@ %% Notifications, which defines a mechanism for an asynchronous %% message notification delivery service for the netconf protocol. %% -%% Specific functions to support this are {@link -%% create_subscription/6} and {@link get_event_streams/3}. (The -%% functions also exist with other arities.) +%% Specific functions to support this are create_subscription/6 +%% get_event_streams/3. (The functions also exist with other arities.) %% -%% @end %%---------------------------------------------------------------------- -module(ct_netconfc). @@ -167,7 +69,13 @@ %%---------------------------------------------------------------------- %% External exports %%---------------------------------------------------------------------- --export([open/1, +-export([connect/1, + connect/2, + disconnect/1, + session/1, + session/2, + session/3, + open/1, open/2, only_open/1, only_open/2, @@ -205,6 +113,7 @@ create_subscription/4, create_subscription/5, create_subscription/6, + get_event_streams/1, get_event_streams/2, get_event_streams/3, get_capabilities/1, @@ -215,7 +124,9 @@ %%---------------------------------------------------------------------- %% Exported types %%---------------------------------------------------------------------- --export_type([notification/0]). +-export_type([client/0, + handle/0, + notification/0]). %%---------------------------------------------------------------------- %% Internal exports @@ -273,13 +184,15 @@ host, port = ?DEFAULT_PORT, timeout = ?DEFAULT_TIMEOUT, - name}). + name, + type}). %% Connection reference -record(connection, {reference, % {CM,Ch} host, port, - name}). + name, + type}). %% Pending replies from server -record(pending, {tref, % timer ref (returned from timer:xxx) @@ -291,17 +204,17 @@ %%---------------------------------------------------------------------- %% Type declarations %%---------------------------------------------------------------------- --type client() :: handle() | ct_gen_conn:server_id() | ct_gen_conn:target_name(). --type handle() :: term(). -%% An opaque reference for a connection (netconf session). See {@link -%% ct} for more information. +-type client() :: handle() | server_id() | ct:target_name(). +-opaque handle() :: pid(). -type options() :: [option()]. -%% Options used for setting up ssh connection to a netconf server. - -type option() :: {ssh,host()} | {port,inet:port_number()} | {user,string()} | {password,string()} | {user_dir,string()} | {timeout,timeout()}. + +-type session_options() :: [session_option()]. +-type session_option() :: {timeout,timeout()}. + -type host() :: inet:hostname() | inet:ip_address(). -type notification() :: {notification, xml_attributes(), notification_content()}. @@ -317,14 +230,13 @@ %% See XML Schema for Event Notifications found in RFC5277 for further %% detail about the data format for the string values. -%-type error_handler() :: module(). -type error_reason() :: term(). +-type server_id() :: atom(). + -type simple_xml() :: {xml_tag(), xml_attributes(), xml_content()} | {xml_tag(), xml_content()} | xml_tag(). -%% <p>This type is further described in the documentation for the -%% <tt>Xmerl</tt> application.</p> -type xml_tag() :: atom(). -type xml_attributes() :: [{xml_attribute_tag(),xml_attribute_value()}]. -type xml_attribute_tag() :: atom(). @@ -336,69 +248,130 @@ -type xs_datetime() :: string(). %% This date and time identifyer has the same format as the XML type %% dateTime and compliant to RFC3339. The format is -%% ```[-]CCYY-MM-DDThh:mm:ss[.s][Z|(+|-)hh:mm]''' +%% "[-]CCYY-MM-DDThh:mm:ss[.s][Z|(+|-)hh:mm]" %%---------------------------------------------------------------------- %% External interface functions %%---------------------------------------------------------------------- %%---------------------------------------------------------------------- --spec open(Options) -> Result when +%% Open an SSH connection to a Netconf server +%% If the server options are specified in a configuration file, use +%% open/2. +-spec connect(Options) -> Result when Options :: options(), Result :: {ok,handle()} | {error,error_reason()}. -%% @doc Open a netconf session and exchange `hello' messages. -%% -%% If the server options are specified in a configuration file, or if -%% a named client is needed for logging purposes (see {@section -%% Logging}) use {@link open/2} instead. -%% -%% The opaque `handler()' reference which is returned from this -%% function is required as client identifier when calling any other -%% function in this module. -%% -%% The `timeout' option (milli seconds) is used when setting up -%% the ssh connection and when waiting for the hello message from the -%% server. It is not used for any other purposes during the lifetime -%% of the connection. -%% -%% @end +connect(Options) -> + do_connect(Options, #options{type=connection},[]). + +-spec connect(KeyOrName,ExtraOptions) -> Result when + KeyOrName :: ct:key_or_name(), + ExtraOptions :: options(), + Result :: {ok,handle()} | {error,error_reason()}. +connect(KeyOrName, ExtraOptions) -> + SortedExtra = lists:keysort(1,ExtraOptions), + SortedConfig = lists:keysort(1,ct:get_config(KeyOrName,[])), + AllOpts = lists:ukeymerge(1,SortedConfig,SortedExtra), + do_connect(AllOpts,#options{name=KeyOrName,type=connection},[{name,KeyOrName}]). + +do_connect(OptList,InitOptRec,NameOpt) -> + case check_options(OptList,InitOptRec) of + {Host,Port,Options} -> + ct_gen_conn:start({Host,Port},Options,?MODULE, + NameOpt ++ [{reconnect,false}, + {use_existing_connection,false}, + {forward_messages,false}]); + Error -> + Error + end. + %%---------------------------------------------------------------------- -open(Options) -> - open(Options,#options{},[],true). +%% Close the given SSH connection. +-spec disconnect(Conn) -> ok | {error,error_reason()} when + Conn :: handle(). +disconnect(Conn) -> + case call(Conn,get_ssh_connection) of + {ok,_} -> + ct_gen_conn:stop(Conn); + Error -> + Error + end. %%---------------------------------------------------------------------- +%% Open a netconf session as a channel on the given SSH connection, +%% and exchange `hello' messages. +-spec session(Conn) -> Result when + Conn :: handle(), + Result :: {ok,handle()} | {error,error_reason()}. +session(Conn) -> + do_session(Conn,[],#options{type=channel},[]). + +-spec session(Conn,Options) -> Result when + Conn :: handle(), + Options :: session_options(), + Result :: {ok,handle()} | {error,error_reason()}; + (KeyOrName,Conn) -> Result when + KeyOrName :: ct:key_or_name(), + Conn :: handle(), + Result :: {ok,handle()} | {error,error_reason()}. +session(Conn,Options) when is_list(Options) -> + do_session(Conn,Options,#options{type=channel},[]); +session(KeyOrName,Conn) -> + do_session(Conn,[],#options{name=KeyOrName,type=channel},[{name,KeyOrName}]). + +-spec session(KeyOrName,Conn,Options) -> Result when + Conn :: handle(), + Options :: session_options(), + KeyOrName :: ct:key_or_name(), + Result :: {ok,handle()} | {error,error_reason()}. +session(KeyOrName,Conn,ExtraOptions) -> + SortedExtra = lists:keysort(1,ExtraOptions), + SortedConfig = lists:keysort(1,ct:get_config(KeyOrName,[])), + AllOpts = lists:ukeymerge(1,SortedConfig,SortedExtra), + do_session(Conn,AllOpts,#options{name=KeyOrName,type=channel}, + [{name,KeyOrName}]). + +do_session(Conn,OptList,InitOptRec,NameOpt) -> + case call(Conn,get_ssh_connection) of + {ok,SshConn} -> + case check_session_options(OptList,InitOptRec) of + {ok,Options} -> + case ct_gen_conn:start(SshConn,Options,?MODULE, + NameOpt ++ + [{reconnect,false}, + {use_existing_connection,false}, + {forward_messages,true}]) of + {ok,Client} -> + case hello(Client,Options#options.timeout) of + ok -> + {ok,Client}; + Error -> + Error + end; + Error -> + Error + end; + Error -> + Error + end; + Error -> + Error + end. + +%%---------------------------------------------------------------------- +%% Open a netconf session and exchange 'hello' messages. +%% If the server options are specified in a configuration file, use +%% open/2. +-spec open(Options) -> Result when + Options :: options(), + Result :: {ok,handle()} | {error,error_reason()}. +open(Options) -> + open(Options,#options{type=connection_and_channel},[],true). + -spec open(KeyOrName, ExtraOptions) -> Result when - KeyOrName :: ct_gen_conn:key_or_name(), + KeyOrName :: ct:key_or_name(), ExtraOptions :: options(), Result :: {ok,handle()} | {error,error_reason()}. -%% @doc Open a named netconf session and exchange `hello' messages. -%% -%% If `KeyOrName' is a configured `server_id()' or a -%% `target_name()' associated with such an ID, then the options -%% for this server will be fetched from the configuration file. -% -%% The `ExtraOptions' argument will be added to the options found in -%% the configuration file. If the same options are given, the values -%% from the configuration file will overwrite `ExtraOptions'. -%% -%% If the server is not specified in a configuration file, use {@link -%% open/1} instead. -%% -%% The opaque `handle()' reference which is returned from this -%% function can be used as client identifier when calling any other -%% function in this module. However, if `KeyOrName' is a -%% `target_name()', i.e. if the server is named via a call to -%% `ct:require/2' or a `require' statement in the test -%% suite, then this name may be used instead of the `handle()'. -%% -%% The `timeout' option (milli seconds) is used when setting up -%% the ssh connection and when waiting for the hello message from the -%% server. It is not used for any other purposes during the lifetime -%% of the connection. -%% -%% @see ct:require/2 -%% @end -%%---------------------------------------------------------------------- open(KeyOrName, ExtraOpts) -> open(KeyOrName, ExtraOpts, true). @@ -406,10 +379,11 @@ open(KeyOrName, ExtraOpts, Hello) -> SortedExtra = lists:keysort(1,ExtraOpts), SortedConfig = lists:keysort(1,ct:get_config(KeyOrName,[])), AllOpts = lists:ukeymerge(1,SortedConfig,SortedExtra), - open(AllOpts,#options{name=KeyOrName},[{name,KeyOrName}],Hello). + open(AllOpts,#options{name=KeyOrName,type=connection_and_channel}, + [{name,KeyOrName}],Hello). open(OptList,InitOptRec,NameOpt,Hello) -> - case check_options(OptList,undefined,undefined,InitOptRec) of + case check_options(OptList,InitOptRec) of {Host,Port,Options} -> case ct_gen_conn:start({Host,Port},Options,?MODULE, NameOpt ++ [{reconnect,false}, @@ -431,296 +405,206 @@ open(OptList,InitOptRec,NameOpt,Hello) -> %%---------------------------------------------------------------------- +%% As open/1,2, except no 'hello' message is sent. -spec only_open(Options) -> Result when Options :: options(), Result :: {ok,handle()} | {error,error_reason()}. -%% @doc Open a netconf session, but don't send `hello'. -%% -%% As {@link open/1} but does not send a `hello' message. -%% -%% @end -%%---------------------------------------------------------------------- only_open(Options) -> - open(Options,#options{},[],false). + open(Options,#options{type=connection_and_channel},[],false). -%%---------------------------------------------------------------------- -spec only_open(KeyOrName,ExtraOptions) -> Result when - KeyOrName :: ct_gen_conn:key_or_name(), + KeyOrName :: ct:key_or_name(), ExtraOptions :: options(), Result :: {ok,handle()} | {error,error_reason()}. -%% @doc Open a name netconf session, but don't send `hello'. -%% -%% As {@link open/2} but does not send a `hello' message. -%% -%% @end -%%---------------------------------------------------------------------- only_open(KeyOrName, ExtraOpts) -> open(KeyOrName, ExtraOpts, false). %%---------------------------------------------------------------------- -%% @spec hello(Client) -> Result -%% @equiv hello(Client, [], infinity) +%% Send a 'hello' message. +-spec hello(Client) -> Result when + Client :: handle(), + Result :: ok | {error,error_reason()}. hello(Client) -> hello(Client,[],?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec hello(Client,Timeout) -> Result when Client :: handle(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @spec hello(Client, Timeout) -> Result -%% @equiv hello(Client, [], Timeout) hello(Client,Timeout) -> hello(Client,[],Timeout). -%%---------------------------------------------------------------------- -spec hello(Client,Options,Timeout) -> Result when Client :: handle(), Options :: [{capability, [string()]}], Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Exchange `hello' messages with the server. -%% -%% Adds optional capabilities and sends a `hello' message to the -%% server and waits for the return. -%% @end -%%---------------------------------------------------------------------- hello(Client,Options,Timeout) -> call(Client, {hello, Options, Timeout}). %%---------------------------------------------------------------------- -%% @spec get_session_id(Client) -> Result -%% @equiv get_session_id(Client, infinity) +%% Get the session id for the session specified by Client. +-spec get_session_id(Client) -> Result when + Client :: client(), + Result :: pos_integer() | {error,error_reason()}. get_session_id(Client) -> get_session_id(Client, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec get_session_id(Client, Timeout) -> Result when Client :: client(), Timeout :: timeout(), Result :: pos_integer() | {error,error_reason()}. -%% @doc Returns the session id associated with the given client. -%% -%% @end -%%---------------------------------------------------------------------- get_session_id(Client, Timeout) -> call(Client, get_session_id, Timeout). %%---------------------------------------------------------------------- -%% @spec get_capabilities(Client) -> Result -%% @equiv get_capabilities(Client, infinity) +%% Get the server side capabilities. +-spec get_capabilities(Client) -> Result when + Client :: client(), + Result :: [string()] | {error,error_reason()}. get_capabilities(Client) -> get_capabilities(Client, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec get_capabilities(Client, Timeout) -> Result when Client :: client(), Timeout :: timeout(), Result :: [string()] | {error,error_reason()}. -%% @doc Returns the server side capabilities -%% -%% The following capability identifiers, defined in RFC 4741, can be returned: -%% -%% <ul> -%% <li>`"urn:ietf:params:netconf:base:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:writable-running:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:candidate:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:confirmed-commit:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:rollback-on-error:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:startup:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:url:1.0"'</li> -%% <li>`"urn:ietf:params:netconf:capability:xpath:1.0"'</li> -%% </ul> -%% -%% Note, additional identifiers may exist, e.g. server side namespace. -%% -%% @end -%%---------------------------------------------------------------------- get_capabilities(Client, Timeout) -> call(Client, get_capabilities, Timeout). %%---------------------------------------------------------------------- -%% @spec send(Client, SimpleXml) -> Result -%% @equiv send(Client, SimpleXml, infinity) +%% Send an XML document to the server. +-spec send(Client, SimpleXml) -> Result when + Client :: client(), + SimpleXml :: simple_xml(), + Result :: simple_xml() | {error,error_reason()}. send(Client, SimpleXml) -> send(Client, SimpleXml, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec send(Client, SimpleXml, Timeout) -> Result when Client :: client(), SimpleXml :: simple_xml(), Timeout :: timeout(), Result :: simple_xml() | {error,error_reason()}. -%% @doc Send an XML document to the server. -%% -%% The given XML document is sent as is to the server. This function -%% can be used for sending XML documents that can not be expressed by -%% other interface functions in this module. send(Client, SimpleXml, Timeout) -> call(Client,{send, Timeout, SimpleXml}). %%---------------------------------------------------------------------- -%% @spec send_rpc(Client, SimpleXml) -> Result -%% @equiv send_rpc(Client, SimpleXml, infinity) +%% Wrap the given XML document in a valid netconf 'rpc' request and +%% send to the server. +-spec send_rpc(Client, SimpleXml) -> Result when + Client :: client(), + SimpleXml :: simple_xml(), + Result :: [simple_xml()] | {error,error_reason()}. send_rpc(Client, SimpleXml) -> send_rpc(Client, SimpleXml, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec send_rpc(Client, SimpleXml, Timeout) -> Result when Client :: client(), SimpleXml :: simple_xml(), Timeout :: timeout(), Result :: [simple_xml()] | {error,error_reason()}. -%% @doc Send a Netconf <code>rpc</code> request to the server. -%% -%% The given XML document is wrapped in a valid Netconf -%% <code>rpc</code> request and sent to the server. The -%% <code>message-id</code> and namespace attributes are added to the -%% <code>rpc</code> element. -%% -%% This function can be used for sending <code>rpc</code> requests -%% that can not be expressed by other interface functions in this -%% module. send_rpc(Client, SimpleXml, Timeout) -> call(Client,{send_rpc, SimpleXml, Timeout}). %%---------------------------------------------------------------------- -%% @spec lock(Client, Target) -> Result -%% @equiv lock(Client, Target, infinity) +%% Send a 'lock' request. +-spec lock(Client, Target) -> Result when + Client :: client(), + Target :: netconf_db(), + Result :: ok | {error,error_reason()}. lock(Client, Target) -> lock(Client, Target,?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec lock(Client, Target, Timeout) -> Result when Client :: client(), Target :: netconf_db(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Unlock configuration target. -%% -%% Which target parameters that can be used depends on if -%% `:candidate' and/or `:startup' are supported by the -%% server. If successfull, the configuration system of the device is -%% not available to other clients (Netconf, CORBA, SNMP etc). Locks -%% are intended to be short-lived. -%% -%% The operations {@link kill_session/2} or {@link kill_session/3} can -%% be used to force the release of a lock owned by another Netconf -%% session. How this is achieved by the server side is implementation -%% specific. -%% -%% @end -%%---------------------------------------------------------------------- lock(Client, Target, Timeout) -> call(Client,{send_rpc_op,lock,[Target],Timeout}). %%---------------------------------------------------------------------- -%% @spec unlock(Client, Target) -> Result -%% @equiv unlock(Client, Target, infinity) +%% Send a 'unlock' request. +-spec unlock(Client, Target) -> Result when + Client :: client(), + Target :: netconf_db(), + Result :: ok | {error,error_reason()}. unlock(Client, Target) -> unlock(Client, Target,?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec unlock(Client, Target, Timeout) -> Result when Client :: client(), Target :: netconf_db(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Unlock configuration target. -%% -%% If the client earlier has aquired a lock, via {@link lock/2} or -%% {@link lock/3}, this operation release the associated lock. To be -%% able to access another target than `running', the server must -%% support `:candidate' and/or `:startup'. -%% -%% @end -%%---------------------------------------------------------------------- unlock(Client, Target, Timeout) -> call(Client, {send_rpc_op, unlock, [Target], Timeout}). %%---------------------------------------------------------------------- -%% @spec get(Client, Filter) -> Result -%% @equiv get(Client, Filter, infinity) +%% Send a 'get' request. +-spec get(Client, Filter) -> Result when + Client :: client(), + Filter :: simple_xml() | xpath(), + Result :: {ok,[simple_xml()]} | {error,error_reason()}. get(Client, Filter) -> get(Client, Filter, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec get(Client, Filter, Timeout) -> Result when Client :: client(), Filter :: simple_xml() | xpath(), Timeout :: timeout(), Result :: {ok,[simple_xml()]} | {error,error_reason()}. -%% @doc Get data. -%% -%% This operation returns both configuration and state data from the -%% server. -%% -%% Filter type `xpath' can only be used if the server supports -%% `:xpath'. -%% -%% @end -%%---------------------------------------------------------------------- get(Client, Filter, Timeout) -> call(Client,{send_rpc_op, get, [Filter], Timeout}). %%---------------------------------------------------------------------- -%% @spec get_config(Client, Source, Filter) -> Result -%% @equiv get_config(Client, Source, Filter, infinity) +%% Send a 'get-config' request. +-spec get_config(Client, Source, Filter) -> Result when + Client :: client(), + Source :: netconf_db(), + Filter :: simple_xml() | xpath(), + Result :: {ok,[simple_xml()]} | {error,error_reason()}. get_config(Client, Source, Filter) -> get_config(Client, Source, Filter, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec get_config(Client, Source, Filter, Timeout) -> Result when Client :: client(), Source :: netconf_db(), Filter :: simple_xml() | xpath(), Timeout :: timeout(), Result :: {ok,[simple_xml()]} | {error,error_reason()}. -%% @doc Get configuration data. -%% -%% To be able to access another source than `running', the server -%% must advertise `:candidate' and/or `:startup'. -%% -%% Filter type `xpath' can only be used if the server supports -%% `:xpath'. -%% -%% -%% @end -%%---------------------------------------------------------------------- get_config(Client, Source, Filter, Timeout) -> call(Client, {send_rpc_op, get_config, [Source, Filter], Timeout}). %%---------------------------------------------------------------------- -%% @spec edit_config(Client, Target, Config) -> Result -%% @equiv edit_config(Client, Target, Config, [], infinity) +%% Send a 'edit-config' request. +-spec edit_config(Client, Target, Config) -> Result when + Client :: client(), + Target :: netconf_db(), + Config :: simple_xml(), + Result :: ok | {error,error_reason()}. edit_config(Client, Target, Config) -> edit_config(Client, Target, Config, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- --spec edit_config(Client, Target, Config, OptParamsOrTimeout) -> Result when +-spec edit_config(Client, Target, Config, OptParams) -> Result when + Client :: client(), + Target :: netconf_db(), + Config :: simple_xml(), + OptParams :: [simple_xml()], + Result :: ok | {error,error_reason()}; + (Client, Target, Config, Timeout) -> Result when Client :: client(), Target :: netconf_db(), Config :: simple_xml(), - OptParamsOrTimeout :: [simple_xml()] | timeout(), + Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc -%% -%% If `OptParamsOrTimeout' is a timeout value, then this is -%% equivalent to {@link edit_config/5. edit_config(Client, Target, -%% Config, [], Timeout)}. -%% -%% If `OptParamsOrTimeout' is a list of simple XML, then this is -%% equivalent to {@link edit_config/5. edit_config(Client, Target, -%% Config, OptParams, infinity)}. -%% -%% @end edit_config(Client, Target, Config, Timeout) when ?is_timeout(Timeout) -> edit_config(Client, Target, Config, [], Timeout); edit_config(Client, Target, Config, OptParams) when is_list(OptParams) -> edit_config(Client, Target, Config, OptParams, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec edit_config(Client, Target, Config, OptParams, Timeout) -> Result when Client :: client(), Target :: netconf_db(), @@ -728,99 +612,79 @@ edit_config(Client, Target, Config, OptParams) when is_list(OptParams) -> OptParams :: [simple_xml()], Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Edit configuration data. -%% -%% Per default only the running target is available, unless the server -%% include `:candidate' or `:startup' in its list of -%% capabilities. -%% -%% `OptParams' can be used for specifying optional parameters -%% (`default-operation', `test-option' or `error-option') that will be -%% added to the `edit-config' request. The value must be a list -%% containing valid simple XML, for example -%% -%% ``` -%% [{'default-operation', ["none"]}, -%% {'error-option', ["rollback-on-error"]}] -%%''' -%% -%% @end -%%---------------------------------------------------------------------- edit_config(Client, Target, Config, OptParams, Timeout) -> call(Client, {send_rpc_op, edit_config, [Target,Config,OptParams], Timeout}). %%---------------------------------------------------------------------- -%% @spec delete_config(Client, Target) -> Result -%% @equiv delete_config(Client, Target, infinity) +%% Send a 'delete-config' request. +-spec delete_config(Client, Target) -> Result when + Client :: client(), + Target :: startup | candidate, + Result :: ok | {error,error_reason()}. delete_config(Client, Target) -> delete_config(Client, Target, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec delete_config(Client, Target, Timeout) -> Result when Client :: client(), Target :: startup | candidate, Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Delete configuration data. -%% -%% The running configuration cannot be deleted and `:candidate' -%% or `:startup' must be advertised by the server. -%% -%% @end -%%---------------------------------------------------------------------- delete_config(Client, Target, Timeout) when Target == startup; Target == candidate -> call(Client,{send_rpc_op, delete_config, [Target], Timeout}). %%---------------------------------------------------------------------- -%% @spec copy_config(Client, Source, Target) -> Result -%% @equiv copy_config(Client, Source, Target, infinity) +%% Send a 'copy-config' request. +-spec copy_config(Client, Target, Source) -> Result when + Client :: client(), + Target :: netconf_db(), + Source :: netconf_db(), + Result :: ok | {error,error_reason()}. copy_config(Client, Source, Target) -> copy_config(Client, Source, Target, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec copy_config(Client, Target, Source, Timeout) -> Result when Client :: client(), Target :: netconf_db(), Source :: netconf_db(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Copy configuration data. -%% -%% Which source and target options that can be issued depends on the -%% capabilities supported by the server. I.e. `:candidate' and/or -%% `:startup' are required. -%% -%% @end -%%---------------------------------------------------------------------- copy_config(Client, Target, Source, Timeout) -> call(Client,{send_rpc_op, copy_config, [Target, Source], Timeout}). %%---------------------------------------------------------------------- -%% @spec action(Client, Action) -> Result -%% @equiv action(Client, Action, infinity) +%% Execute an action. +-spec action(Client, Action) -> Result when + Client :: client(), + Action :: simple_xml(), + Result :: ok | {ok,[simple_xml()]} | {error,error_reason()}. action(Client,Action) -> action(Client,Action,?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec action(Client, Action, Timeout) -> Result when Client :: client(), Action :: simple_xml(), Timeout :: timeout(), Result :: ok | {ok,[simple_xml()]} | {error,error_reason()}. -%% @doc Execute an action. If the return type is void, <c>ok</c> will -%% be returned instead of <c>{ok,[simple_xml()]}</c>. -%% -%% @end -%%---------------------------------------------------------------------- action(Client,Action,Timeout) -> call(Client,{send_rpc_op, action, [Action], Timeout}). %%---------------------------------------------------------------------- +%% Send a 'create-subscription' request +%% See RFC5277, NETCONF Event Notifications +-spec create_subscription(Client) -> Result when + Client :: client(), + Result :: ok | {error,error_reason()}. create_subscription(Client) -> create_subscription(Client,?DEFAULT_STREAM,?DEFAULT_TIMEOUT). +-spec create_subscription(Client, Stream | Filter | Timeout) -> Result when + Client :: client(), + Stream :: stream_name(), + Filter :: simple_xml() | [simple_xml()], + Timeout :: timeout(), + Result :: ok | {error,error_reason()}. create_subscription(Client,Timeout) when ?is_timeout(Timeout) -> create_subscription(Client,?DEFAULT_STREAM,Timeout); @@ -876,6 +740,22 @@ create_subscription(Client,Stream,Filter,Timeout) [Stream,Filter,undefined,undefined], Timeout}). +-spec create_subscription(Client, Stream, StartTime, StopTime, Timeout) -> + Result when + Client :: client(), + Stream :: stream_name(), + StartTime :: xs_datetime(), + StopTime :: xs_datetime(), + Timeout :: timeout(), + Result :: ok | {error,error_reason()}; + (Client, Stream, Filter,StartTime, StopTime) -> + Result when + Client :: client(), + Stream :: stream_name(), + Filter :: simple_xml() | [simple_xml()], + StartTime :: xs_datetime(), + StopTime :: xs_datetime(), + Result :: ok | {error,error_reason()}. create_subscription(Client,Stream,StartTime,StopTime,Timeout) when ?is_string(Stream) andalso ?is_string(StartTime) andalso @@ -891,7 +771,6 @@ create_subscription(Client,Stream,Filter,StartTime,StopTime) ?is_string(StopTime) -> create_subscription(Client,Stream,Filter,StartTime,StopTime,?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec create_subscription(Client, Stream, Filter,StartTime, StopTime, Timeout) -> Result when Client :: client(), @@ -901,168 +780,75 @@ create_subscription(Client,Stream,Filter,StartTime,StopTime) StopTime :: xs_datetime(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Create a subscription for event notifications. -%% -%% This function sets up a subscription for netconf event -%% notifications of the given stream type, matching the given -%% filter. The calling process will receive notifications as messages -%% of type `notification()'. -%% -%% <dl> -%% <dt>Stream:</dt> -%% <dd> An optional parameter that indicates which stream of events -%% is of interest. If not present, events in the default NETCONF -%% stream will be sent.</dd> -%% -%% <dt>Filter:</dt> -%% <dd>An optional parameter that indicates which subset of all -%% possible events is of interest. The format of this parameter is -%% the same as that of the filter parameter in the NETCONF protocol -%% operations. If not present, all events not precluded by other -%% parameters will be sent.</dd> -%% -%% <dt>StartTime:</dt> -%% <dd>An optional parameter used to trigger the replay feature and -%% indicate that the replay should start at the time specified. If -%% `StartTime' is not present, this is not a replay subscription. -%% It is not valid to specify start times that are later than the -%% current time. If the `StartTime' specified is earlier than the -%% log can support, the replay will begin with the earliest -%% available notification. This parameter is of type dateTime and -%% compliant to [RFC3339]. Implementations must support time -%% zones.</dd> -%% -%% <dt>StopTime:</dt> -%% <dd>An optional parameter used with the optional replay feature -%% to indicate the newest notifications of interest. If `StopTime' -%% is not present, the notifications will continue until the -%% subscription is terminated. Must be used with and be later than -%% `StartTime'. Values of `StopTime' in the future are valid. This -%% parameter is of type dateTime and compliant to [RFC3339]. -%% Implementations must support time zones.</dd> -%% </dl> -%% -%% See RFC5277 for further details about the event notification -%% mechanism. -%% -%% @end -%%---------------------------------------------------------------------- create_subscription(Client,Stream,Filter,StartTime,StopTime,Timeout) -> call(Client,{send_rpc_op,{create_subscription, self()}, [Stream,Filter,StartTime,StopTime], Timeout}). %%---------------------------------------------------------------------- -%% @spec get_event_streams(Client, Timeout) -> Result -%% @equiv get_event_streams(Client, [], Timeout) +%% Send a request to get the given event streams +%% See RFC5277, NETCONF Event Notifications +-spec get_event_streams(Client) + -> Result when + Client :: client(), + Result :: {ok,streams()} | {error,error_reason()}. +get_event_streams(Client) -> + get_event_streams(Client,[],?DEFAULT_TIMEOUT). + +-spec get_event_streams(Client, Timeout) + -> Result when + Client :: client(), + Timeout :: timeout(), + Result :: {ok,streams()} | {error,error_reason()}; + (Client, Streams) -> Result when + Client :: client(), + Streams :: [stream_name()], + Result :: {ok,streams()} | {error,error_reason()}. get_event_streams(Client,Timeout) when is_integer(Timeout); Timeout==infinity -> get_event_streams(Client,[],Timeout); - -%%---------------------------------------------------------------------- -%% @spec get_event_streams(Client, Streams) -> Result -%% @equiv get_event_streams(Client, Streams, infinity) get_event_streams(Client,Streams) when is_list(Streams) -> get_event_streams(Client,Streams,?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec get_event_streams(Client, Streams, Timeout) -> Result when Client :: client(), Streams :: [stream_name()], Timeout :: timeout(), Result :: {ok,streams()} | {error,error_reason()}. -%% @doc Send a request to get the given event streams. -%% -%% `Streams' is a list of stream names. The following filter will -%% be sent to the netconf server in a `get' request: -%% -%% ``` -%% <netconf xmlns="urn:ietf:params:xml:ns:netmod:notification"> -%% <streams> -%% <stream> -%% <name>StreamName1</name> -%% </stream> -%% <stream> -%% <name>StreamName2</name> -%% </stream> -%% ... -%% </streams> -%% </netconf> -%% ''' -%% -%% If `Streams' is an empty list, ALL streams will be requested -%% by sending the following filter: -%% -%% ``` -%% <netconf xmlns="urn:ietf:params:xml:ns:netmod:notification"> -%% <streams/> -%% </netconf> -%% ''' -%% -%% If more complex filtering is needed, a use {@link get/2} or {@link -%% get/3} and specify the exact filter according to XML Schema for -%% Event Notifications found in RFC5277. -%% -%% @end -%%---------------------------------------------------------------------- get_event_streams(Client,Streams,Timeout) -> call(Client,{get_event_streams,Streams,Timeout}). %%---------------------------------------------------------------------- -%% @spec close_session(Client) -> Result -%% @equiv close_session(Client, infinity) +%% Send a 'close-session' request +-spec close_session(Client) -> Result when + Client :: client(), + Result :: ok | {error,error_reason()}. close_session(Client) -> close_session(Client, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec close_session(Client, Timeout) -> Result when Client :: client(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Request graceful termination of the session associated with the client. -%% -%% When a netconf server receives a `close-session' request, it -%% will gracefully close the session. The server will release any -%% locks and resources associated with the session and gracefully -%% close any associated connections. Any NETCONF requests received -%% after a `close-session' request will be ignored. -%% -%% @end -%%---------------------------------------------------------------------- close_session(Client, Timeout) -> call(Client,{send_rpc_op, close_session, [], Timeout}, true). %%---------------------------------------------------------------------- -%% @spec kill_session(Client, SessionId) -> Result -%% @equiv kill_session(Client, SessionId, infinity) +%% Send a 'kill-session' request +-spec kill_session(Client, SessionId) -> Result when + Client :: client(), + SessionId :: pos_integer(), + Result :: ok | {error,error_reason()}. kill_session(Client, SessionId) -> kill_session(Client, SessionId, ?DEFAULT_TIMEOUT). -%%---------------------------------------------------------------------- -spec kill_session(Client, SessionId, Timeout) -> Result when Client :: client(), SessionId :: pos_integer(), Timeout :: timeout(), Result :: ok | {error,error_reason()}. -%% @doc Force termination of the session associated with the supplied -%% session id. -%% -%% The server side shall abort any operations currently in process, -%% release any locks and resources associated with the session, and -%% close any associated connections. -%% -%% Only if the server is in the confirmed commit phase, the -%% configuration will be restored to its state before entering the -%% confirmed commit phase. Otherwise, no configuration roll back will -%% be performed. -%% -%% If the given `SessionId' is equal to the current session id, -%% an error will be returned. -%% -%% @end -%% ---------------------------------------------------------------------- kill_session(Client, SessionId, Timeout) -> call(Client,{send_rpc_op, kill_session, [SessionId], Timeout}). @@ -1071,24 +857,35 @@ kill_session(Client, SessionId, Timeout) -> %% Callback functions %%---------------------------------------------------------------------- -%% @private +init(_KeyOrName,{CM,{Host,Port}},Options) -> + case ssh_channel(#connection{reference=CM,host=Host,port=Port},Options) of + {ok,Connection} -> + {ok, CM, #state{connection = Connection}}; + {error,Reason}-> + {error,Reason} + end; +init(_KeyOrName,{_Host,_Port},Options) when Options#options.type==connection -> + case ssh_connect(Options) of + {ok, Connection} -> + ConnPid = Connection#connection.reference, + {ok, ConnPid, #state{connection = Connection}}; + Error -> + Error + end; init(_KeyOrName,{_Host,_Port},Options) -> case ssh_open(Options) of {ok, Connection} -> - log(Connection,open), {ConnPid,_} = Connection#connection.reference, {ok, ConnPid, #state{connection = Connection}}; {error,Reason}-> {error,Reason} end. -%% @private + terminate(_, #state{connection=Connection}) -> ssh_close(Connection), - log(Connection,close), ok. -%% @private handle_msg({hello, Options, Timeout}, From, #state{connection=Connection,hello_status=HelloStatus} = State) -> case do_send(Connection, client_hello(Options)) of @@ -1107,6 +904,14 @@ handle_msg({hello, Options, Timeout}, From, Error -> {stop, Error, State} end; +handle_msg(get_ssh_connection, _From, #state{connection=Connection}=State) -> + Reply = + case Connection#connection.reference of + {_,_} -> {error,not_an_ssh_connection}; + CM -> {ok,{CM,{Connection#connection.host, + Connection#connection.port}}} + end, + {reply, Reply, State}; handle_msg(_, _From, #state{session_id=undefined} = State) -> %% Hello is not yet excanged - this shall never happen {reply,{error,waiting_for_hello},State}; @@ -1136,7 +941,6 @@ handle_msg({get_event_streams=Op,Streams,Timeout}, From, State) -> SimpleXml = encode_rpc_operation(get,[Filter]), do_send_rpc(Op, SimpleXml, Timeout, From, State). -%% @private handle_msg({ssh_cm, CM, {data, Ch, _Type, Data}}, State) -> ssh_connection:adjust_window(CM,Ch,size(Data)), handle_data(Data, State); @@ -1172,7 +976,6 @@ handle_msg({Ref,timeout},#state{pending=Pending} = State) -> %% the implementation before this patch {R,State#state{pending=Pending1, no_end_tag_buff= <<>>, buff= <<>>}}. -%% @private %% Called by ct_util_server to close registered connections before terminate. close(Client) -> case get_handle(Client) of @@ -1243,15 +1046,18 @@ get_handle(Client) -> Error end. +check_options(OptList,Options) -> + check_options(OptList,undefined,undefined,Options). + check_options([], undefined, _Port, _Options) -> {error, no_host_address}; check_options([], _Host, undefined, _Options) -> {error, no_port}; check_options([], Host, Port, Options) -> {Host,Port,Options}; -check_options([{ssh, Host}|T], _, Port, #options{} = Options) -> +check_options([{ssh, Host}|T], _, Port, Options) -> check_options(T, Host, Port, Options#options{host=Host}); -check_options([{port,Port}|T], Host, _, #options{} = Options) -> +check_options([{port,Port}|T], Host, _, Options) -> check_options(T, Host, Port, Options#options{port=Port}); check_options([{timeout, Timeout}|T], Host, Port, Options) when is_integer(Timeout); Timeout==infinity -> @@ -1262,6 +1068,15 @@ check_options([Opt|T], Host, Port, #options{ssh=SshOpts}=Options) -> %% Option verified by ssh check_options(T, Host, Port, Options#options{ssh=[Opt|SshOpts]}). +check_session_options([],Options) -> + {ok,Options}; +check_session_options([{timeout, Timeout}|T], Options) + when is_integer(Timeout); Timeout==infinity -> + check_session_options(T, Options#options{timeout = Timeout}); +check_session_options([Opt|_T], _Options) -> + {error, {invalid_option, Opt}}. + + %%%----------------------------------------------------------------- set_request_timer(infinity) -> {undefined,undefined}; @@ -1356,7 +1171,6 @@ do_send_rpc(Connection, MsgId, SimpleXml) -> do_send(Connection, SimpleXml) -> Xml=to_xml_doc(SimpleXml), - log(Connection,send,Xml), ssh_send(Connection, Xml). to_xml_doc(Simple) -> @@ -1766,9 +1580,14 @@ decode_streams([]) -> log(Connection,Action) -> log(Connection,Action,<<>>). -log(#connection{host=Host,port=Port,name=Name},Action,Data) -> +log(#connection{reference=Ref,host=Host,port=Port,name=Name},Action,Data) -> + Address = + case Ref of + {_,Ch} -> {Host,Port,Ch}; + _ -> {Host,Port} + end, error_logger:info_report(#conn_log{client=self(), - address={Host,Port}, + address=Address, name=Name, action=Action, module=?MODULE}, @@ -1776,7 +1595,6 @@ log(#connection{host=Host,port=Port,name=Name},Action,Data) -> %% Log callback - called from the error handler process -%% @private format_data(How,Data) -> %% Assuming that the data is encoded as UTF-8. If it is not, then %% the printout might be wrong, but the format function will not @@ -1915,42 +1733,84 @@ get_tag([]) -> %%%----------------------------------------------------------------- %%% SSH stuff - -ssh_open(#options{host=Host,timeout=Timeout,port=Port,ssh=SshOpts,name=Name}) -> +ssh_connect(#options{host=Host,timeout=Timeout,port=Port, + ssh=SshOpts,name=Name,type=Type}) -> case ssh:connect(Host, Port, [{user_interaction,false}, - {silently_accept_hosts, true}|SshOpts]) of + {silently_accept_hosts, true}|SshOpts], + Timeout) of {ok,CM} -> - case ssh_connection:session_channel(CM, Timeout) of - {ok,Ch} -> - case ssh_connection:subsystem(CM, Ch, "netconf", Timeout) of - success -> - {ok, #connection{reference = {CM,Ch}, - host = Host, - port = Port, - name = Name}}; - failure -> - ssh:close(CM), - {error,{ssh,could_not_execute_netconf_subsystem}}; - {error,timeout} -> - {error,{ssh,could_not_execute_netconf_subsystem,timeout}} - end; - {error, Reason} -> - ssh:close(CM), - {error,{ssh,could_not_open_channel,Reason}} - end; + Connection = #connection{reference = CM, + host = Host, + port = Port, + name = Name, + type = Type}, + log(Connection,connect), + {ok,Connection}; {error,Reason} -> {error,{ssh,could_not_connect_to_server,Reason}} end. -ssh_send(#connection{reference = {CM,Ch}}, Data) -> +ssh_channel(#connection{reference=CM}=Connection0, + #options{timeout=Timeout,name=Name,type=Type}) -> + case ssh_connection:session_channel(CM, Timeout) of + {ok,Ch} -> + case ssh_connection:subsystem(CM, Ch, "netconf", Timeout) of + success -> + Connection = Connection0#connection{reference = {CM,Ch}, + name = Name, + type = Type}, + log(Connection,open), + {ok, Connection}; + failure -> + ssh_connection:close(CM,Ch), + {error,{ssh,could_not_execute_netconf_subsystem}}; + {error,timeout} -> + ssh_connection:close(CM,Ch), + {error,{ssh,could_not_execute_netconf_subsystem,timeout}} + end; + {error, Reason} -> + {error,{ssh,could_not_open_channel,Reason}} + end. + + +ssh_open(Options) -> + case ssh_connect(Options) of + {ok,Connection} -> + case ssh_channel(Connection,Options) of + {ok,_} = Ok -> + Ok; + Error -> + ssh_close(Connection), + Error + end; + Error -> + Error + end. + +ssh_send(#connection{reference = {CM,Ch}}=Connection, Data) -> case ssh_connection:send(CM, Ch, Data) of - ok -> ok; - {error,Reason} -> {error,{ssh,failed_to_send_data,Reason}} + ok -> + log(Connection,send,Data), + ok; + {error,Reason} -> + {error,{ssh,failed_to_send_data,Reason}} end. -ssh_close(#connection{reference = {CM,_Ch}}) -> - ssh:close(CM). +ssh_close(Connection=#connection{reference = {CM,Ch}, type = Type}) -> + _ = ssh_connection:close(CM,Ch), + log(Connection,close), + case Type of + connection_and_channel -> + ssh_close(Connection#connection{reference = CM}); + _ -> + ok + end, + ok; +ssh_close(Connection=#connection{reference = CM}) -> + _ = ssh:close(CM), + log(Connection,disconnect), + ok. %%---------------------------------------------------------------------- diff --git a/lib/common_test/src/ct_release_test.erl b/lib/common_test/src/ct_release_test.erl index d783f8d04e..6e3cad0a50 100644 --- a/lib/common_test/src/ct_release_test.erl +++ b/lib/common_test/src/ct_release_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2014-2016. All Rights Reserved. +%% Copyright Ericsson AB 2014-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -132,7 +132,7 @@ %%----------------------------------------------------------------- -define(testnode, 'ct_release_test-upgrade'). --define(exclude_apps, [hipe, typer, dialyzer]). % never include these apps +-define(exclude_apps, [hipe, dialyzer]). % never include these apps %%----------------------------------------------------------------- -record(ct_data, {from,to}). diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index fbb9c7ab60..ce30babc0d 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2016. All Rights Reserved. +%% Copyright Ericsson AB 2004-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -76,8 +76,8 @@ abort_if_missing_suites, silent_connections = [], stylesheet, - multiply_timetraps = 1, - scale_timetraps = false, + multiply_timetraps, + scale_timetraps, create_priv_dir, testspec_files = [], current_testspec, @@ -264,11 +264,11 @@ script_start1(Parent, Args) -> [], Args), Verbosity = verbosity_args2opts(Args), MultTT = get_start_opt(multiply_timetraps, - fun([MT]) -> list_to_integer(MT) end, 1, Args), + fun([MT]) -> list_to_integer(MT) end, Args), ScaleTT = get_start_opt(scale_timetraps, fun([CT]) -> list_to_atom(CT); ([]) -> true - end, false, Args), + end, Args), CreatePrivDir = get_start_opt(create_priv_dir, fun([PD]) -> list_to_atom(PD); ([]) -> auto_per_tc @@ -363,6 +363,12 @@ script_start1(Parent, Args) -> _ -> application:set_env(common_test, disable_log_cache, true) end, + %% log_cleanup - used by ct_logs + KeepLogs = get_start_opt(keep_logs, + fun ct_logs:parse_keep_logs/1, + all, + Args), + application:set_env(common_test, keep_logs, KeepLogs), Opts = #opts{label = Label, profile = Profile, vts = Vts, shell = Shell, @@ -430,13 +436,17 @@ script_start2(Opts = #opts{vts = undefined, Specs1 = get_start_opt(join_specs, [Specs], Specs, Args), %% using testspec as input for test Relaxed = get_start_opt(allow_user_terms, true, false, Args), - case catch ct_testspec:collect_tests_from_file(Specs1, Relaxed) of - {E,Reason} when E == error ; E == 'EXIT' -> + try ct_testspec:collect_tests_from_file(Specs1, Relaxed) of + TestSpecData -> + execute_all_specs(TestSpecData, Opts, Args, []) + catch + throw:{error,Reason} -> StackTrace = erlang:get_stacktrace(), {error,{invalid_testspec,{Reason,StackTrace}}}; - TestSpecData -> - execute_all_specs(TestSpecData, Opts, Args, []) - end; + _:Reason -> + StackTrace = erlang:get_stacktrace(), + {error,{invalid_testspec,{Reason,StackTrace}}} + end; [] -> {error,no_testspec_specified}; _ -> % no testspec used @@ -970,6 +980,12 @@ run_test1(StartOpts) when is_list(StartOpts) -> stop_trace(Tracing), exit(Res); RefreshDir -> + %% log_cleanup - used by ct_logs + KeepLogs = get_start_opt(keep_logs, + fun ct_logs:parse_keep_logs/1, + all, + StartOpts), + application:set_env(common_test, keep_logs, KeepLogs), ok = refresh_logs(?abs(RefreshDir)), exit(done) end. @@ -1055,8 +1071,8 @@ run_test2(StartOpts) -> CoverStop = get_start_opt(cover_stop, value, StartOpts), %% timetrap manipulation - MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts), - ScaleTT = get_start_opt(scale_timetraps, value, false, StartOpts), + MultiplyTT = get_start_opt(multiply_timetraps, value, StartOpts), + ScaleTT = get_start_opt(scale_timetraps, value, StartOpts), %% create unique priv dir names CreatePrivDir = get_start_opt(create_priv_dir, value, StartOpts), @@ -1131,6 +1147,12 @@ run_test2(StartOpts) -> DisableCacheBool -> application:set_env(common_test, disable_log_cache, DisableCacheBool) end, + %% log_cleanup - used by ct_logs + KeepLogs = get_start_opt(keep_logs, + fun ct_logs:parse_keep_logs/1, + all, + StartOpts), + application:set_env(common_test, keep_logs, KeepLogs), %% stepped execution Step = get_start_opt(step, value, StartOpts), @@ -1180,12 +1202,16 @@ run_spec_file(Relaxed, end, AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts), - case catch ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of - {Error,CTReason} when Error == error ; Error == 'EXIT' -> - StackTrace = erlang:get_stacktrace(), - exit({error,{invalid_testspec,{CTReason,StackTrace}}}); + try ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of TestSpecData -> run_all_specs(TestSpecData, Opts, StartOpts, []) + catch + throw:{error,CTReason} -> + StackTrace = erlang:get_stacktrace(), + exit({error,{invalid_testspec,{CTReason,StackTrace}}}); + _:CTReason -> + StackTrace = erlang:get_stacktrace(), + exit({error,{invalid_testspec,{CTReason,StackTrace}}}) end. run_all_specs([], _, _, TotResult) -> @@ -2155,8 +2181,8 @@ continue(_MakeErrors, true) -> false; continue(_MakeErrors, _AbortIfMissingSuites) -> io:nl(), - OldGl = group_leader(), - case set_group_leader_same_as_shell() of + OldGL = group_leader(), + case set_group_leader_same_as_shell(OldGL) of true -> S = self(), io:format("Failed to compile or locate one " @@ -2172,7 +2198,7 @@ continue(_MakeErrors, _AbortIfMissingSuites) -> S ! false end end), - group_leader(OldGl, self()), + group_leader(OldGL, self()), receive R when R==true; R==false -> R after 15000 -> @@ -2184,7 +2210,9 @@ continue(_MakeErrors, _AbortIfMissingSuites) -> true end. -set_group_leader_same_as_shell() -> +set_group_leader_same_as_shell(OldGL) -> + %% find the group leader process on the node in a dirty fashion + %% (check initial function call and look in the process dictionary) GS2or3 = fun(P) -> case process_info(P,initial_call) of {initial_call,{group,server,X}} when X == 2 ; X == 3 -> @@ -2197,7 +2225,10 @@ set_group_leader_same_as_shell() -> true == lists:keymember(shell,1, element(2,process_info(P,dictionary)))] of [GL|_] -> - group_leader(GL, self()); + %% check if started from remote node (skip interaction) + if node(OldGL) /= node(GL) -> false; + true -> group_leader(GL, self()) + end; [] -> false end. @@ -2275,8 +2306,19 @@ do_run_test(Tests, Skip, Opts0) -> _Lower -> ok end, - test_server_ctrl:multiply_timetraps(Opts0#opts.multiply_timetraps), - test_server_ctrl:scale_timetraps(Opts0#opts.scale_timetraps), + + case Opts0#opts.multiply_timetraps of + undefined -> MultTT = 1; + MultTT -> MultTT + end, + case Opts0#opts.scale_timetraps of + undefined -> ScaleTT = false; + ScaleTT -> ScaleTT + end, + ct_logs:log("TEST INFO","Timetrap time multiplier = ~w~n" + "Timetrap scaling enabled = ~w", [MultTT,ScaleTT]), + test_server_ctrl:multiply_timetraps(MultTT), + test_server_ctrl:scale_timetraps(ScaleTT), test_server_ctrl:create_priv_dir(choose_val( Opts0#opts.create_priv_dir, diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index 571958ca03..4188bd7c3b 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -309,7 +309,12 @@ is_started(ENode) -> % make a Erlang node name from name and hostname enodename(Host, Node) -> - list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)). + case lists:member($@, atom_to_list(Node)) of + true -> + Node; + false -> + list_to_atom(atom_to_list(Node)++"@"++atom_to_list(Host)) + end. % performs actual start of the "slave" node do_start(Host, Node, Options) -> diff --git a/lib/common_test/src/ct_snmp.erl b/lib/common_test/src/ct_snmp.erl index 2c59b19196..5844909d17 100644 --- a/lib/common_test/src/ct_snmp.erl +++ b/lib/common_test/src/ct_snmp.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2015. All Rights Reserved. +%% Copyright Ericsson AB 2004-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/common_test/src/ct_ssh.erl b/lib/common_test/src/ct_ssh.erl index 6ab3bf036c..5ac91f396b 100644 --- a/lib/common_test/src/ct_ssh.erl +++ b/lib/common_test/src/ct_ssh.erl @@ -68,7 +68,8 @@ send_and_receive/3, send_and_receive/4, send_and_receive/5, send_and_receive/6, exec/2, exec/3, exec/4, - subsystem/3, subsystem/4]). + subsystem/3, subsystem/4, + shell/2, shell/3]). %% STFP Functions -export([sftp_connect/1, @@ -94,6 +95,7 @@ -record(state, {ssh_ref, conn_type, target}). +-type handle() :: pid(). %%%----------------------------------------------------------------- %%%------------------------ SSH COMMANDS --------------------------- @@ -490,6 +492,22 @@ subsystem(SSH, ChannelId, Subsystem, Timeout) -> call(SSH, {subsystem,ChannelId,Subsystem,Timeout}). +-spec shell(SSH, ChannelId) -> Result when + SSH :: handle() | ct:target_name(), + ChannelId :: ssh:ssh_channel_id(), + Result :: ok | {error,term()}. +shell(SSH, ChannelId) -> + shell(SSH, ChannelId, ?DEFAULT_TIMEOUT). + +-spec shell(SSH, ChannelId, Timeout) -> Result when + SSH :: handle() | ct:target_name(), + ChannelId :: ssh:ssh_channel_id(), + Timeout :: timeout(), + Result :: ok | {error,term()}. +shell(SSH, ChannelId, Timeout) -> + call(SSH, {shell,ChannelId,Timeout}). + + %%%----------------------------------------------------------------- %%%------------------------ SFTP COMMANDS -------------------------- @@ -1067,6 +1085,14 @@ handle_msg({subsystem,Chn,Subsystem,TO}, State) -> Result = ssh_connection:subsystem(SSHRef, Chn, Subsystem, TO), {Result,State}; +handle_msg({shell,Chn,TO}, State) -> + #state{ssh_ref=SSHRef, target=Target} = State, + try_log(heading(shell,Target), + "SSH Ref: ~p, Chn: ~p, Timeout: ~p", + [SSHRef,Chn,TO]), + Result = ssh_connection:shell(SSHRef, Chn), + {Result,State}; + %% --- SFTP Commands --- handle_msg({read_file,Srv,File}=Cmd, S=#state{ssh_ref=SSHRef}) -> diff --git a/lib/common_test/src/ct_telnet.erl b/lib/common_test/src/ct_telnet.erl index 34d27ed5f4..bff1112ab9 100644 --- a/lib/common_test/src/ct_telnet.erl +++ b/lib/common_test/src/ct_telnet.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2015. All Rights Reserved. +%% Copyright Ericsson AB 2003-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 991abb0666..180786273d 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2016. All Rights Reserved. +%% Copyright Ericsson AB 2006-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -26,7 +26,8 @@ -export([prepare_tests/1, prepare_tests/2, collect_tests_from_list/2, collect_tests_from_list/3, - collect_tests_from_file/2, collect_tests_from_file/3]). + collect_tests_from_file/2, collect_tests_from_file/3, + get_tests/1]). -export([testspec_rec2list/1, testspec_rec2list/2]). @@ -803,6 +804,31 @@ list_nodes(#testspec{nodes=NodeRefs}) -> lists:map(fun({_Ref,Node}) -> Node end, NodeRefs). +%%%----------------------------------------------------------------- +%%% Parse the given test specs and return the complete set of specs +%%% and tests to run/skip. +%%% [Spec1,Spec2,...] means create separate tests per spec +%%% [[Spec1,Spec2,...]] means merge all specs into one +-spec get_tests(Specs) -> {ok,[{Specs,Tests}]} | {error,Reason} when + Specs :: [string()] | [[string()]], + Tests :: {Node,Run,Skip}, + Node :: atom(), + Run :: {Dir,Suites,Cases}, + Skip :: {Dir,Suites,Comment} | {Dir,Suites,Cases,Comment}, + Dir :: string(), + Suites :: atom | [atom()] | all, + Cases :: atom | [atom()] | all, + Comment :: string(), + Reason :: term(). + +get_tests(Specs) -> + case collect_tests_from_file(Specs,true) of + Tests when is_list(Tests) -> + {ok,[{S,prepare_tests(R)} || {S,R} <- Tests]}; + Error -> + Error + end. + %% ----------------------------------------------------- %% / \ %% | When adding test/config terms, remember to update | @@ -1132,6 +1158,11 @@ handle_data(verbosity,Node,VLvls,_Spec) when is_list(VLvls) -> VLvls1 = lists:map(fun(VLvl = {_Cat,_Lvl}) -> VLvl; (Lvl) -> {'$unspecified',Lvl} end, VLvls), [{Node,VLvls1}]; +handle_data(multiply_timetraps,Node,Mult,_Spec) when is_integer(Mult) -> + [{Node,Mult}]; +handle_data(scale_timetraps,Node,Scale,_Spec) when Scale == true; + Scale == false -> + [{Node,Scale}]; handle_data(silent_connections,Node,all,_Spec) -> [{Node,[all]}]; handle_data(silent_connections,Node,Conn,_Spec) when is_atom(Conn) -> @@ -1150,6 +1181,8 @@ should_be_added(Tag,Node,_Data,Spec) -> Tag == label; Tag == auto_compile; Tag == abort_if_missing_suites; Tag == stylesheet; Tag == verbosity; + Tag == multiply_timetraps; + Tag == scale_timetraps; Tag == silent_connections -> lists:keymember(ref2node(Node,Spec#testspec.nodes),1, read_field(Spec,Tag)) == false; diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index 82a8743cf0..802e9be97c 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -188,6 +188,8 @@ do_start(Parent, Mode, LogDir, Verbosity) -> ok end, + ct_default_gl:start_link(group_leader()), + {StartTime,TestLogDir} = ct_logs:init(Mode, Verbosity), ct_event:notify(#event{name=test_start, @@ -199,14 +201,7 @@ do_start(Parent, Mode, LogDir, Verbosity) -> ok -> Parent ! {self(),started}; {fail,CTHReason} -> - ErrorInfo = if is_atom(CTHReason) -> - io_lib:format("{~p,~p}", - [CTHReason, - erlang:get_stacktrace()]); - true -> - CTHReason - end, - ct_logs:tc_print('Suite Callback',ErrorInfo,[]), + ct_logs:tc_print('Suite Callback',CTHReason,[]), self() ! {{stop,{self(),{user_error,CTHReason}}}, {Parent,make_ref()}} catch @@ -474,6 +469,7 @@ loop(Mode,TestData,StartDir) -> ct_logs:close(Info, StartDir), ct_event:stop(), ct_config:stop(), + ct_default_gl:stop(), ok = file:set_cwd(StartDir), return(From, Info); {Ref, _Msg} when is_reference(Ref) -> @@ -926,7 +922,8 @@ warn_duplicates(Suites) -> [] -> ok; _ -> - io:format(user,"~nWARNING! Deprecated function: ~w:sequences/0.~n" + io:format(?def_gl, + "~nWARNING! Deprecated function: ~w:sequences/0.~n" " Use group with sequence property instead.~n",[Mod]) end end, @@ -980,12 +977,12 @@ get_profile_data(Profile, Key, StartDir) -> end, case Result of {error,enoent} when Profile /= default -> - io:format(user, "~nERROR! Missing profile file ~p~n", [File]), + io:format(?def_gl, "~nERROR! Missing profile file ~p~n", [File]), undefined; {error,enoent} when Profile == default -> undefined; {error,Reason} -> - io:format(user,"~nERROR! Error in profile file ~p: ~p~n", + io:format(?def_gl,"~nERROR! Error in profile file ~p: ~p~n", [WhichFile,Reason]), undefined; {ok,Data} -> @@ -995,7 +992,7 @@ get_profile_data(Profile, Key, StartDir) -> _ when is_list(Data) -> Data; _ -> - io:format(user, + io:format(?def_gl, "~nERROR! Invalid profile data in ~p~n", [WhichFile]), [] @@ -1082,10 +1079,10 @@ open_url(iexplore, Args, URL) -> Path = proplists:get_value(default, Paths), [Cmd | _] = string:tokens(Path, "%"), Cmd1 = Cmd ++ " " ++ Args ++ " " ++ URL, - io:format(user, "~nOpening ~ts with command:~n ~ts~n", [URL,Cmd1]), + io:format(?def_gl, "~nOpening ~ts with command:~n ~ts~n", [URL,Cmd1]), open_port({spawn,Cmd1}, []); _ -> - io:format("~nNo path to iexplore.exe~n",[]) + io:format(?def_gl, "~nNo path to iexplore.exe~n",[]) end, win32reg:close(R), ok; @@ -1095,6 +1092,6 @@ open_url(Prog, Args, URL) -> is_list(Prog) -> Prog end, Cmd = ProgStr ++ " " ++ Args ++ " " ++ URL, - io:format(user, "~nOpening ~ts with command:~n ~ts~n", [URL,Cmd]), + io:format(?def_gl, "~nOpening ~ts with command:~n ~ts~n", [URL,Cmd]), open_port({spawn,Cmd},[]), ok. diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index d7efa26863..039c8168ec 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -23,6 +23,7 @@ -define(board_table,ct_boards). -define(suite_table,ct_suite_data). -define(verbosity_table,ct_verbosity_table). +-define(def_gl, ct_default_gl). -record(conn, {handle, targetref, diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index 883da0da0a..faff150ac9 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -24,11 +24,11 @@ %% %% suite() -> %% [{ct_hooks, [{cth_conn_log, -%% [{ct_netconfc:conn_mod(),ct_netconfc:hook_options()}]}]}]. +%% [{conn_mod(),hook_options()}]}]}]. %% %% or specified in a configuration file: %% -%% {ct_conn_log,[{ct_netconfc:conn_mod(),ct_netconfc:hook_options()}]}. +%% {ct_conn_log,[{conn_mod(),hook_options()}]}. %% %% The conn_mod() is the common test module implementing the protocol, %% e.g. ct_netconfc, ct_telnet, etc. This module must log by calling @@ -54,32 +54,21 @@ -include_lib("common_test/include/ct.hrl"). -export([init/2, - pre_init_per_testcase/3, - post_end_per_testcase/4]). - -%%---------------------------------------------------------------------- -%% Exported types -%%---------------------------------------------------------------------- --export_type([hook_options/0, - log_type/0, - conn_mod/0]). + pre_init_per_testcase/4, + post_end_per_testcase/5]). %%---------------------------------------------------------------------- %% Type declarations %%---------------------------------------------------------------------- --type hook_options() :: [hook_option()]. -%% Options that can be given to `cth_conn_log' in the `ct_hook' statement. --type hook_option() :: {log_type,log_type()} | - {hosts,[ct_gen_conn:key_or_name()]}. --type log_type() :: raw | pretty | html | silent. --type conn_mod() :: ct_netconfc | ct_telnet. +-type hook_options() :: ct:conn_log_options(). +-type log_type() :: ct:conn_log_type(). +-type conn_mod() :: ct:conn_log_mod(). %%---------------------------------------------------------------------- -spec init(Id, HookOpts) -> Result when Id :: term(), HookOpts :: hook_options(), - Result :: {ok,[{conn_mod(), - {log_type(),[ct_gen_conn:key_or_name()]}}]}. + Result :: {ok,[{conn_mod(),{log_type(),[ct:key_or_name()]}}]}. init(_Id, HookOpts) -> ConfOpts = ct:get_config(ct_conn_log,[]), {ok,merge_log_info(ConfOpts,HookOpts)}. @@ -104,7 +93,7 @@ get_log_opts(Mod,Opts) -> Hosts = proplists:get_value(hosts,Opts,[]), {LogType,Hosts}. -pre_init_per_testcase(TestCase,Config,CthState) -> +pre_init_per_testcase(_Suite,TestCase,Config,CthState) -> Logs = lists:map( fun({ConnMod,{LogType,Hosts}}) -> @@ -158,7 +147,7 @@ pre_init_per_testcase(TestCase,Config,CthState) -> ct_util:update_testdata(?MODULE, Update, [create]), {Config,CthState}. -post_end_per_testcase(TestCase,_Config,Return,CthState) -> +post_end_per_testcase(_Suite,TestCase,_Config,Return,CthState) -> Update = fun(PrevUsers) -> case lists:delete(TestCase, PrevUsers) of diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 6d77d7ee9e..1bc9b10d41 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011-2016. All Rights Reserved. +%% Copyright Ericsson AB 2011-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -28,10 +28,10 @@ %% CTH Callbacks -export([id/1, init/2, pre_init_per_suite/3, pre_end_per_suite/3, post_end_per_suite/4, - pre_init_per_group/3, post_init_per_group/4, - pre_end_per_group/3, post_end_per_group/4, - pre_init_per_testcase/3, post_init_per_testcase/4, - pre_end_per_testcase/3, post_end_per_testcase/4]). + pre_init_per_group/4, post_init_per_group/5, + pre_end_per_group/4, post_end_per_group/5, + pre_init_per_testcase/4, post_init_per_testcase/5, + pre_end_per_testcase/4, post_end_per_testcase/5]). %% Event handler Callbacks -export([init/1, @@ -71,11 +71,11 @@ post_end_per_suite(_Suite, Config, Return, State) -> set_curr_func(undefined, Config), {Return, State}. -pre_init_per_group(Group, Config, State) -> +pre_init_per_group(_Suite, Group, Config, State) -> set_curr_func({group,Group,init_per_group}, Config), {Config, State}. -post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) -> +post_init_per_group(_Suite, Group, Config, Result, tc_log_async) when is_list(Config) -> case lists:member(parallel,proplists:get_value( tc_group_properties,Config,[])) of true -> @@ -83,33 +83,33 @@ post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) -> false -> {Result, tc_log_async} end; -post_init_per_group(_Group, _Config, Result, State) -> +post_init_per_group(_Suite, _Group, _Config, Result, State) -> {Result, State}. -pre_init_per_testcase(TC, Config, State) -> +pre_init_per_testcase(_Suite, TC, Config, State) -> set_curr_func(TC, Config), {Config, State}. -post_init_per_testcase(_TC, _Config, Return, State) -> +post_init_per_testcase(_Suite, _TC, _Config, Return, State) -> {Return, State}. -pre_end_per_testcase(_TC, Config, State) -> +pre_end_per_testcase(_Suite, _TC, Config, State) -> {Config, State}. -post_end_per_testcase(_TC, _Config, Result, State) -> +post_end_per_testcase(_Suite, _TC, _Config, Result, State) -> %% Make sure that the event queue is flushed %% before ending this test case. gen_event:call(error_logger, ?MODULE, flush, 300000), {Result, State}. -pre_end_per_group(Group, Config, {tc_log, Group}) -> +pre_end_per_group(_Suite, Group, Config, {tc_log, Group}) -> set_curr_func({group,Group,end_per_group}, Config), {Config, set_log_func(tc_log_async)}; -pre_end_per_group(Group, Config, State) -> +pre_end_per_group(_Suite, Group, Config, State) -> set_curr_func({group,Group,end_per_group}, Config), {Config, State}. -post_end_per_group(_Group, Config, Return, State) -> +post_end_per_group(_Suite, _Group, Config, Return, State) -> set_curr_func({group,undefined}, Config), {Return, State}. diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 59b916851e..0bbb217275 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -33,16 +33,16 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([pre_end_per_group/4]). +-export([post_end_per_group/5]). --export([pre_init_per_testcase/3]). --export([post_end_per_testcase/4]). +-export([pre_init_per_testcase/4]). +-export([post_end_per_testcase/5]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). +-export([on_tc_fail/4]). +-export([on_tc_skip/4]). -export([terminate/1]). @@ -116,29 +116,29 @@ pre_end_per_suite(_Suite,Config,State) -> post_end_per_suite(_Suite,Config,Result,State) -> {Result, end_tc(end_per_suite,Config,Result,State)}. -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(_Suite,Group,Config,State) -> {Config, init_tc(State#state{ curr_group = [Group|State#state.curr_group]}, Config)}. -post_init_per_group(_Group,Config,Result,State) -> +post_init_per_group(_Suite,_Group,Config,Result,State) -> {Result, end_tc(init_per_group,Config,Result,State)}. -pre_end_per_group(_Group,Config,State) -> +pre_end_per_group(_Suite,_Group,Config,State) -> {Config, init_tc(State, Config)}. -post_end_per_group(_Group,Config,Result,State) -> +post_end_per_group(_Suite,_Group,Config,Result,State) -> NewState = end_tc(end_per_group, Config, Result, State), {Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}. -pre_init_per_testcase(_TC,Config,State) -> +pre_init_per_testcase(_Suite,_TC,Config,State) -> {Config, init_tc(State, Config)}. -post_end_per_testcase(TC,Config,Result,State) -> +post_end_per_testcase(_Suite,TC,Config,Result,State) -> {Result, end_tc(TC,Config, Result,State)}. -on_tc_fail(_TC, _Res, State = #state{test_cases = []}) -> +on_tc_fail(_Suite,_TC, _Res, State = #state{test_cases = []}) -> State; -on_tc_fail(_TC, Res, State) -> +on_tc_fail(_Suite,_TC, Res, State) -> TCs = State#state.test_cases, TC = hd(TCs), NewTC = TC#testcase{ @@ -146,10 +146,9 @@ on_tc_fail(_TC, Res, State) -> {fail,lists:flatten(io_lib:format("~p",[Res]))} }, State#state{ test_cases = [NewTC | tl(TCs)]}. -on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0) - when Type == tc_auto_skip; Type == tc_user_skip -> - on_tc_skip(ConfigFunc, Res, State0); -on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> +on_tc_skip(Suite,{ConfigFunc,_GrName}, Res, State) -> + on_tc_skip(Suite,ConfigFunc, Res, State); +on_tc_skip(Suite,Tc, Res, State0) -> TcStr = atom_to_list(Tc), State = case State0#state.test_cases of @@ -158,11 +157,7 @@ on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> _ -> State0 end, - do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[]))); -on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) -> - State; -on_tc_skip(_Tc, Res, State) -> - do_tc_skip(Res, State). + do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(set_suite(Suite,State),[]))). do_tc_skip(Res, State) -> TCs = State#state.test_cases, @@ -209,6 +204,12 @@ end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite, result = passed }| State#state.test_cases], tc_log = ""}. % so old tc_log is not set if next is on_tc_skip + +set_suite(Suite,#state{curr_suite=undefined}=State) -> + State#state{curr_suite=Suite, curr_suite_ts=?now}; +set_suite(_,State) -> + State. + close_suite(#state{ test_cases = [] } = State) -> State; close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> @@ -228,7 +229,8 @@ close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> testcases = lists:reverse(TCs), log = SuiteLog, url = SuiteUrl}, - State#state{ test_cases = [], + State#state{ curr_suite = undefined, + test_cases = [], test_suites = [Suite | State#state.test_suites]}. terminate(State = #state{ test_cases = [] }) -> diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 924086f2bd..a4a84393c1 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2016. All Rights Reserved. +%% Copyright Ericsson AB 1996-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -359,10 +359,10 @@ stick_all_sticky(Node,Sticky) -> %% cover. run_test_case_apply({Mod,Func,Args,Name,RunInit,TimetrapData}) -> - case os:getenv("TS_RUN_VALGRIND") of + case is_valgrind() of false -> ok; - _ -> + true -> os:putenv("VALGRIND_LOGFILE_INFIX",atom_to_list(Mod)++"."++ atom_to_list(Func)++"-") end, @@ -778,9 +778,9 @@ spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid, %% if init_per_testcase fails, the test case %% should be skipped try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why), - do_init_tc_call(Mod,{end_per_testcase,Func}, + do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, [CurrConf],{ok,[CurrConf]}), - do_end_tc_call(Mod,{end_per_testcase,Func}, + do_end_tc_call(Mod,{end_per_testcase_not_run,Func}, {Pid,Skip,[CurrConf]}, Why) end of _ -> ok catch @@ -1151,14 +1151,14 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) -> Args end, EPTCInitRes = - case do_init_tc_call(Mod,{end_per_testcase,Func}, + case do_init_tc_call(Mod,{end_per_testcase_not_run,Func}, IPTCEndRes,Return) of {ok,EPTCInitConfig} when is_list(EPTCInitConfig) -> {Return,EPTCInitConfig}; _ -> - Return + {Return,IPTCEndRes} end, - do_end_tc_call1(Mod, {end_per_testcase,Func}, + do_end_tc_call1(Mod, {end_per_testcase_not_run,Func}, EPTCInitRes, Return); _Ok -> do_end_tc_call1(Mod, IPTC, Res, Return) @@ -1827,7 +1827,8 @@ timetrap_scale_factor() -> { 2, fun() -> has_lock_checking() end}, { 3, fun() -> has_superfluous_schedulers() end}, { 6, fun() -> is_debug() end}, - {10, fun() -> is_cover() end} + {10, fun() -> is_cover() end}, + {10, fun() -> is_valgrind() end} ]). timetrap_scale_factor(Scales) -> @@ -2729,6 +2730,16 @@ is_commercial() -> _ -> true end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% is_valgrind() -> boolean() +%% +%% Returns true if valgrind is running, else false +is_valgrind() -> + case os:getenv("TS_RUN_VALGRIND") of + false -> false; + _ -> true + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% Apply given function and reply to caller or proxy. diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index b52e4bef9b..064e375cd5 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2016. All Rights Reserved. +%% Copyright Ericsson AB 2002-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -2051,17 +2051,21 @@ add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) -> add_init_and_end_per_suite([], LastMod, LastRef, FwMod) -> %% we'll add end_per_suite here even if it's not exported %% (and simply let the call fail if it's missing) - case erlang:function_exported(LastMod, end_per_suite, 1) of - true -> - [{conf,LastRef,[],{LastMod,end_per_suite}}]; - false -> + case {erlang:function_exported(LastMod, end_per_suite, 1), + erlang:function_exported(LastMod, init_per_suite, 1)} of + {false,false} -> %% let's call a "fake" end_per_suite if it exists case erlang:function_exported(FwMod, end_per_suite, 1) of true -> [{conf,LastRef,[{suite,LastMod}],{FwMod,end_per_suite}}]; false -> [{conf,LastRef,[],{LastMod,end_per_suite}}] - end + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + [{conf,LastRef,[],{LastMod,end_per_suite}}] end. do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> @@ -2070,11 +2074,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> _ -> ok end, {Init,NextMod,NextRef} = - case erlang:function_exported(Mod, init_per_suite, 1) of - true -> - Ref = make_ref(), - {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref}; - false -> + case {erlang:function_exported(Mod, init_per_suite, 1), + erlang:function_exported(Mod, end_per_suite, 1)} of + {false,false} -> %% let's call a "fake" init_per_suite if it exists case erlang:function_exported(FwMod, init_per_suite, 1) of true -> @@ -2083,8 +2085,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> {FwMod,init_per_suite}}],Mod,Ref}; false -> {[],Mod,undefined} - end - + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + Ref = make_ref(), + {[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref} end, Cases = if LastRef==undefined -> @@ -2094,10 +2101,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> true -> %% we'll add end_per_suite here even if it's not exported %% (and simply let the call fail if it's missing) - case erlang:function_exported(LastMod, end_per_suite, 1) of - true -> - [{conf,LastRef,[],{LastMod,end_per_suite}}|Init]; - false -> + case {erlang:function_exported(LastMod, end_per_suite, 1), + erlang:function_exported(LastMod, init_per_suite, 1)} of + {false,false} -> %% let's call a "fake" end_per_suite if it exists case erlang:function_exported(FwMod, end_per_suite, 1) of true -> @@ -2105,8 +2111,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> {FwMod,end_per_suite}}|Init]; false -> [{conf,LastRef,[],{LastMod,end_per_suite}}|Init] - end - end + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + [{conf,LastRef,[],{LastMod,end_per_suite}}|Init] + end end, {Cases,NextMod,NextRef}. @@ -2115,11 +2126,9 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> No when No==undefined ; No==skipped_suite -> {[],Mod,skipped_suite}; _Ref -> - case erlang:function_exported(LastMod, end_per_suite, 1) of - true -> - {[{conf,LastRef,[],{LastMod,end_per_suite}}], - Mod,skipped_suite}; - false -> + case {erlang:function_exported(LastMod, end_per_suite, 1), + erlang:function_exported(LastMod, init_per_suite, 1)} of + {false,false} -> case erlang:function_exported(FwMod, end_per_suite, 1) of true -> %% let's call "fake" end_per_suite if it exists @@ -2128,7 +2137,13 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) -> false -> {[{conf,LastRef,[],{LastMod,end_per_suite}}], Mod,skipped_suite} - end + end; + _ -> + %% If any of these exist, the other should too + %% (required and documented). If it isn't, it will fail + %% with reason 'undef'. + {[{conf,LastRef,[],{LastMod,end_per_suite}}], + Mod,skipped_suite} end end. @@ -2924,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) exit(framework_error); %% sequential execution of test case finished {Time,RetVal,_} -> + RetTag = + if is_tuple(RetVal) -> element(1,RetVal); + true -> undefined + end, {Failed,Status1} = - case Time of - died -> - {true,update_status(failed, Mod, Func, Status)}; - _ when is_tuple(RetVal) -> - case element(1, RetVal) of - R when R=='EXIT'; R==failed -> - {true,update_status(failed, Mod, Func, Status)}; - R when R==skip; R==skipped -> - {false,update_status(skipped, Mod, Func, Status)}; - _ -> - {false,update_status(ok, Mod, Func, Status)} - end; - _ -> - {false,update_status(ok, Mod, Func, Status)} - end, + case RetTag of + Skip when Skip==skip; Skip==skipped -> + {false,update_status(skipped, Mod, Func, Status)}; + Fail when Fail=='EXIT'; Fail==failed -> + {true,update_status(failed, Mod, Func, Status)}; + _ when Time==died, RetVal=/=ok -> + {true,update_status(failed, Mod, Func, Status)}; + _ -> + {false,update_status(ok, Mod, Func, Status)} + end, case check_prop(sequence, Mode) of false -> stop_minor_log_file(), @@ -3794,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, {died,{timetrap_timeout,TimetrapTimeout}} -> progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); - {died,Reason} -> + {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> + %% died in init_per_testcase + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); + {died,Reason} when Reason=/=ok -> + %% (If Reason==ok it means that process died in + %% end_per_testcase after successfully completing the + %% test case itself - then we shall not fail, but a + %% warning will be issued in the comment field.) progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; @@ -3943,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {ReportTag,Reason1}}]), + TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; + true -> "~w" + end, [Time]), ReasonStr = escape_chars(reason_to_string(Reason1)), ReasonStr1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ReasonStr,[$\n])]), @@ -3957,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, _ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")" end, print(html, - "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" + "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"~ts\">SKIPPED</font></td>" "<td>~ts~ts</td></tr>\n", - [Time,Color,ReasonStr2,Comment1]), + [TimeStr,Color,ReasonStr2,Comment1]), FormatLoc = test_server_sup:format_loc(Loc), print(minor, "=== Location: ~ts", [FormatLoc]), print(minor, "=== Reason: ~ts", [ReasonStr1]), @@ -4098,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, Comment0, {St0,St1}) -> print(minor, "successfully completed test case", []), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]), + TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; + true -> "~w" + end, [Time]), Comment = case RetVal of {comment,RetComment} -> @@ -4116,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, end, print(major, "=elapsed ~p", [Time]), print(html, - "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" + "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"green\">Ok</font></td>" "~ts</tr>\n", - [Time,Comment]), + [TimeStr,Comment]), print(minor, escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])), []), diff --git a/lib/common_test/src/test_server_gl.erl b/lib/common_test/src/test_server_gl.erl index 7d6fe64b92..4845b86dd3 100644 --- a/lib/common_test/src/test_server_gl.erl +++ b/lib/common_test/src/test_server_gl.erl @@ -24,7 +24,7 @@ %% through the test_server_io module/process. -module(test_server_gl). --export([start_link/0,stop/1,set_minor_fd/3,unset_minor_fd/1, +-export([start_link/1,stop/1,set_minor_fd/3,unset_minor_fd/1, get_tc_supervisor/1,print/4,set_props/2]). -export([init/1,handle_call/3,handle_cast/2,handle_info/2,terminate/2]). @@ -33,6 +33,7 @@ tc :: mfa() | 'undefined', %Current test case MFA minor :: 'none'|pid(), %Minor fd minor_monitor, %Monitor ref for minor fd + tsio_monitor, %Monitor red for controlling proc capture :: 'none'|pid(), %Capture output reject_io :: boolean(), %Reject I/O requests... permit_io, %... and exceptions @@ -45,8 +46,8 @@ %% Start a new group leader process. Only to be called by %% the test_server_io process. -start_link() -> - case gen_server:start_link(?MODULE, [], []) of +start_link(TSIO) -> + case gen_server:start_link(?MODULE, [TSIO], []) of {ok,Pid} -> {ok,Pid}; Other -> @@ -130,14 +131,16 @@ set_props(GL, PropList) -> %%% Internal functions. -init([]) -> +init([TSIO]) -> EscChars = case application:get_env(test_server, esc_chars) of {ok,ECBool} -> ECBool; _ -> true end, + Ref = erlang:monitor(process, TSIO), {ok,#st{tc_supervisor=none, minor=none, minor_monitor=none, + tsio_monitor=Ref, capture=none, reject_io=false, permit_io=gb_sets:empty(), @@ -176,6 +179,9 @@ handle_info({'DOWN',Ref,process,_,Reason}=D, #st{minor_monitor=Ref}=St) -> test_server_io:print_unexpected(Data) end, {noreply,St#st{minor=none,minor_monitor=none}}; +handle_info({'DOWN',Ref,process,_,_}, #st{tsio_monitor=Ref}=St) -> + %% controlling process (test_server_io) terminated, we're done + {stop,normal,St}; handle_info({permit_io,Pid}, #st{permit_io=P}=St) -> {noreply,St#st{permit_io=gb_sets:add(Pid, P)}}; handle_info({capture,Cap0}, St) -> diff --git a/lib/common_test/src/test_server_io.erl b/lib/common_test/src/test_server_io.erl index 3d5238052b..fdabf17b08 100644 --- a/lib/common_test/src/test_server_io.erl +++ b/lib/common_test/src/test_server_io.erl @@ -185,7 +185,7 @@ reset_state() -> init([]) -> process_flag(trap_exit, true), Empty = gb_trees:empty(), - {ok,Shared} = test_server_gl:start_link(), + {ok,Shared} = test_server_gl:start_link(self()), {ok,#st{fds=Empty,shared_gl=Shared,gls=gb_sets:empty(), io_buffering=gb_sets:empty(), buffered=Empty, @@ -200,7 +200,7 @@ req(Req) -> gen_server:call(?MODULE, Req, infinity). handle_call({get_gl,false}, _From, #st{gls=Gls,gl_props=Props}=St) -> - {ok,Pid} = test_server_gl:start_link(), + {ok,Pid} = test_server_gl:start_link(self()), test_server_gl:set_props(Pid, Props), {reply,Pid,St#st{gls=gb_sets:insert(Pid, Gls)}}; handle_call({get_gl,true}, _From, #st{shared_gl=Shared}=St) -> @@ -285,7 +285,7 @@ handle_call(reset_state, _From, #st{fds=Fds,tags=Tags,gls=Gls, ok end, Empty = gb_trees:empty(), - {ok,Shared} = test_server_gl:start_link(), + {ok,Shared} = test_server_gl:start_link(self()), {reply,ok,#st{fds=Empty,shared_gl=Shared,gls=gb_sets:empty(), io_buffering=gb_sets:empty(), buffered=Empty, diff --git a/lib/common_test/src/test_server_node.erl b/lib/common_test/src/test_server_node.erl index 0b406c54cc..32d7d14a1b 100644 --- a/lib/common_test/src/test_server_node.erl +++ b/lib/common_test/src/test_server_node.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2016. All Rights Reserved. +%% Copyright Ericsson AB 2002-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -18,11 +18,11 @@ %% %CopyrightEnd% %% -module(test_server_node). --compile(r12). +-compile(r16). %%% %%% The same compiled code for this module must be possible to load -%%% in R12B and later. +%%% in R16B and later. %%% %% Test Controller interface diff --git a/lib/common_test/src/unix_telnet.erl b/lib/common_test/src/unix_telnet.erl index 4897ddb2f8..0f29b2dbb2 100644 --- a/lib/common_test/src/unix_telnet.erl +++ b/lib/common_test/src/unix_telnet.erl @@ -53,8 +53,6 @@ %%% @see ct_telnet -module(unix_telnet). --compile(export_all). - %% Callbacks for ct_telnet.erl -export([connect/7,get_prompt_regexp/0]). -import(ct_telnet,[start_gen_log/1,log/4,end_gen_log/0]). diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index b1eddfedd7..4f3e0e8266 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2016. All Rights Reserved. +# Copyright Ericsson AB 2008-2017. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -70,7 +70,9 @@ MODULES= \ test_server_SUITE \ test_server_test_lib \ ct_release_test_SUITE \ - ct_log_SUITE + ct_log_SUITE \ + ct_SUITE \ + ct_keep_logs_SUITE ERL_FILES= $(MODULES:%=%.erl) HRL_FILES= test_server_test_lib.hrl diff --git a/lib/common_test/test/ct_SUITE.erl b/lib/common_test/test/ct_SUITE.erl new file mode 100644 index 0000000000..eb98c2544f --- /dev/null +++ b/lib/common_test/test/ct_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(ct_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + [{timetrap,{seconds,30}}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [app_file, appup_file]. + +%%%----------------------------------------------------------------- +%%% Test cases + +app_file(_Config) -> + ok = test_server:app_test(common_test), + ok. + +appup_file(_Config) -> + ok = test_server:appup_test(common_test). + diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index fae23484e6..7468ebe9d9 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -1531,17 +1531,17 @@ test_events(config_func_errors) -> {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, - {?eh,test_stats,{0,1,{0,0}}}, + {?eh,test_stats,{0,0,{0,1}}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, - {?eh,test_stats,{0,2,{0,0}}}, + {?eh,test_stats,{1,0,{0,1}}}, [{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, - {?eh,test_stats,{0,3,{0,0}}}, + {?eh,test_stats,{1,0,{0,2}}}, {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}], @@ -1549,7 +1549,7 @@ test_events(config_func_errors) -> {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}}, {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, - {?eh,test_stats,{0,4,{0,0}}}, + {?eh,test_stats,{2,0,{0,2}}}, {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}}, {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}], diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index 690d0af1bb..8ba14e63bc 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -70,22 +70,25 @@ suite() -> all() -> all(suite). -all(suite) -> +all(suite) -> lists:reverse( [ one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init, faulty_cth_exit_in_init, faulty_cth_exit_in_id, - faulty_cth_exit_in_init_scope_suite, minimal_cth, - minimal_and_maximal_cth, faulty_cth_undef, + faulty_cth_exit_in_init_scope_suite, minimal_cth, + minimal_and_maximal_cth, faulty_cth_undef, scope_per_suite_cth, scope_per_group_cth, scope_suite_cth, - scope_per_suite_state_cth, scope_per_group_state_cth, + scope_per_suite_state_cth, scope_per_group_state_cth, scope_suite_state_cth, fail_pre_suite_cth, double_fail_pre_suite_cth, fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, + skip_pre_init_tc_cth, skip_post_suite_cth, recover_post_suite_cth, update_config_cth, - state_update_cth, options_cth, same_id_cth, + state_update_cth, options_cth, same_id_cth, fail_n_skip_with_minimal_cth, prio_cth, no_config, - data_dir, cth_log + no_init_suite_config, no_init_config, no_end_config, + failed_sequence, repeat_force_stop, config_clash, + callbacks_on_skip, fallback, data_dir, cth_log ] ). @@ -96,10 +99,10 @@ all(suite) -> %%%----------------------------------------------------------------- %%% -one_cth(Config) when is_list(Config) -> +one_cth(Config) when is_list(Config) -> do_test(one_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth], Config). -two_cth(Config) when is_list(Config) -> +two_cth(Config) when is_list(Config) -> do_test(two_empty_cth, "ct_cth_empty_SUITE.erl",[empty_cth,empty_cth], Config). @@ -119,13 +122,13 @@ minimal_cth(Config) when is_list(Config) -> minimal_and_maximal_cth(Config) when is_list(Config) -> do_test(minimal_and_maximal_cth, "ct_cth_empty_SUITE.erl", [minimal_cth, empty_cth],Config). - + faulty_cth_undef(Config) when is_list(Config) -> do_test(faulty_cth_undef, "ct_cth_empty_SUITE.erl", [undef_cth],Config). faulty_cth_exit_in_init_scope_suite(Config) when is_list(Config) -> - do_test(faulty_cth_exit_in_init_scope_suite, + do_test(faulty_cth_exit_in_init_scope_suite, "ct_exit_in_init_scope_suite_cth_SUITE.erl", [],Config). @@ -190,6 +193,10 @@ skip_post_suite_cth(Config) when is_list(Config) -> do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl", [skip_post_suite_cth],Config). +skip_pre_init_tc_cth(Config) -> + do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl", + [skip_pre_init_tc_cth],Config). + recover_post_suite_cth(Config) when is_list(Config) -> do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl", [recover_post_suite_cth],Config). @@ -205,7 +212,7 @@ state_update_cth(Config) when is_list(Config) -> options_cth(Config) when is_list(Config) -> do_test(options_cth, "ct_cth_empty_SUITE.erl", [{empty_cth,[test]}],Config). - + same_id_cth(Config) when is_list(Config) -> do_test(same_id_cth, "ct_cth_empty_SUITE.erl", [same_id_cth,same_id_cth],Config). @@ -223,13 +230,24 @@ no_config(Config) when is_list(Config) -> do_test(no_config, "ct_no_config_SUITE.erl", [verify_config_cth],Config). +no_init_suite_config(Config) when is_list(Config) -> + do_test(no_init_suite_config, "ct_no_init_suite_config_SUITE.erl", + [empty_cth],Config). + +no_init_config(Config) when is_list(Config) -> + do_test(no_init_config, "ct_no_init_config_SUITE.erl",[empty_cth],Config). + +no_end_config(Config) when is_list(Config) -> + do_test(no_end_config, "ct_no_end_config_SUITE.erl",[empty_cth],Config). + data_dir(Config) when is_list(Config) -> do_test(data_dir, "ct_data_dir_SUITE.erl", [verify_data_dir_cth],Config). -cth_log(Config) when is_list(Config) -> +cth_log(Config) when is_list(Config) -> %% test that cth_log_redirect writes properly to %% unexpected I/O log + ct:timetrap({minutes,10}), StartOpts = do_test(cth_log, "cth_log_SUITE.erl", [], Config), Logdir = proplists:get_value(logdir, StartOpts), UnexpIoLogs = @@ -253,29 +271,57 @@ cth_log(Config) when is_list(Config) -> end, UnexpIoLogs), ok. +%% OTP-10599 adds the Suite argument as first argument to all hook +%% callbacks that did not have a Suite argument from before. This test +%% checks that ct_hooks will fall back to old versions of callbacks if +%% new versions are not exported. +fallback(Config) -> + do_test(fallback, "all_hook_callbacks_SUITE.erl",[fallback_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped in different ways +callbacks_on_skip(Config) -> + do_test(callbacks_on_skip, {spec,"skip.spec"},[skip_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped due to failed sequence +failed_sequence(Config) -> + do_test(failed_sequence, "seq_SUITE.erl", [skip_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped due to {force_stop,skip_rest} option +repeat_force_stop(Config) -> + do_test(repeat_force_stop, "repeat_SUITE.erl", [skip_cth], Config, ok, 2, + [{force_stop,skip_rest},{duration,"000009"}]). + +%% Test that expected callbacks, and only those, are called when a test +%% are fails due to clash in config alias names +config_clash(Config) -> + do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config). %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- -do_test(Tag, SWC, CTHs, Config) -> - do_test(Tag, SWC, CTHs, Config, ok). -do_test(Tag, SWC, CTHs, Config, {error,_} = Res) -> - do_test(Tag, SWC, CTHs, Config, Res, 1); -do_test(Tag, SWC, CTHs, Config, Res) -> - do_test(Tag, SWC, CTHs, Config, Res, 2). +do_test(Tag, WTT, CTHs, Config) -> + do_test(Tag, WTT, CTHs, Config, ok). +do_test(Tag, WTT, CTHs, Config, {error,_} = Res) -> + do_test(Tag, WTT, CTHs, Config, Res, 1,[]); +do_test(Tag, WTT, CTHs, Config, Res) -> + do_test(Tag, WTT, CTHs, Config, Res, 2,[]). -do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> - +do_test(Tag, WhatToTest, CTHs, Config, Res, EC, ExtraOpts) when is_list(WhatToTest) -> + do_test(Tag, {suite,WhatToTest}, CTHs, Config, Res, EC, ExtraOpts); +do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) -> DataDir = ?config(data_dir, Config), - Suites = filelib:wildcard( - filename:join([DataDir,"cth/tests",SuiteWildCard])), - {Opts,ERPid} = setup([{suite,Suites}, - {ct_hooks,CTHs},{label,Tag}], Config), + Files = filelib:wildcard( + filename:join([DataDir,"cth/tests",Wildcard])), + {Opts,ERPid} = + setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config), Res = ct_test_support:run(Opts, Config), Events = ct_test_support:get_events(ERPid, Config), - ct_test_support:log_events(Tag, + ct_test_support:log_events(Tag, reformat(Events, ?eh), ?config(priv_dir, Config), Opts), @@ -323,12 +369,12 @@ test_events(one_empty_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{empty_cth,post_init_per_testcase,[test_case,'$proplist','_',[]]}}, - {?eh,cth,{empty_cth,pre_end_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, [ct_cth_empty_SUITE,'$proplist',[]]}}, @@ -355,12 +401,12 @@ test_events(two_empty_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, @@ -402,7 +448,7 @@ test_events(minimal_cth) -> {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -423,10 +469,10 @@ test_events(minimal_and_maximal_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_suite,[ct_cth_empty_SUITE,'$proplist','_',[]]}}, @@ -452,11 +498,11 @@ test_events(faulty_cth_undef) -> {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed, FailReason}}}, {?eh,cth,{'_',on_tc_skip,'_'}}, - + {?eh,tc_auto_skip,{ct_cth_empty_SUITE,end_per_suite, {failed, FailReason}}}, {?eh,cth,{'_',on_tc_skip,'_'}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; @@ -512,10 +558,10 @@ test_events(scope_per_suite_cth) -> {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [ct_scope_per_suite_cth_SUITE,'$proplist',[]]}}, @@ -538,10 +584,10 @@ test_events(scope_suite_cth) -> {?eh,tc_done,{ct_scope_suite_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_suite_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_cth_SUITE,'$proplist','_',[]]}}, @@ -561,20 +607,20 @@ test_events(scope_per_group_cth) -> [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, - {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1, '$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, - {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}}, {?eh,cth,{'_',terminate,[[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}], - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -592,10 +638,10 @@ test_events(scope_per_suite_state_cth) -> {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [ct_scope_per_suite_state_cth_SUITE,'$proplist',[test]]}}, @@ -618,10 +664,10 @@ test_events(scope_suite_state_cth) -> {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist',[test]]}}, {?eh,cth,{'_',post_end_per_suite,[ct_scope_suite_state_cth_SUITE,'$proplist','_',[test]]}}, @@ -641,20 +687,20 @@ test_events(scope_per_group_state_cth) -> [{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[test]]}}, {?eh,cth,{'_',init,['_',[test]]}}, - {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}}, + {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','$proplist',[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',ok,[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}}, - {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}}, - {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}}, + {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist',[test]]}}, + {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','_',[test]]}}, {?eh,cth,{'_',terminate,[[test]]}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}], - + {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,end_per_suite}}, {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -666,7 +712,7 @@ test_events(fail_pre_suite_cth) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{'_',init,['_',[]]}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist', @@ -674,27 +720,27 @@ test_events(fail_pre_suite_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, {failed, {error,"Test failure"}}}}, {?eh,cth,{'_',on_tc_fail, - [init_per_suite,{failed,"Test failure"},[]]}}, + [ct_cth_empty_SUITE,init_per_suite,"Test failure",[]]}}, + - {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed,{ct_cth_empty_SUITE,init_per_suite, {failed,"Test failure"}}}}}, {?eh,cth,{'_',on_tc_skip, - [test_case, {tc_auto_skip, + [ct_cth_empty_SUITE,test_case, {tc_auto_skip, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}},[]]}}, - + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}}}, {?eh,cth,{'_',on_tc_skip, - [end_per_suite, {tc_auto_skip, + [ct_cth_empty_SUITE,end_per_suite, {tc_auto_skip, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}},[]]}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, {?eh,stop_logging,[]} @@ -727,17 +773,17 @@ test_events(fail_post_suite_cth) -> {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite, {failed,{error,"Test failure"}}}}, - {?eh,cth,{'_',on_tc_fail,[init_per_suite, {failed,"Test failure"}, []]}}, + {?eh,cth,{'_',on_tc_fail,[ct_cth_empty_SUITE,init_per_suite, "Test failure", []]}}, {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case, {failed,{ct_cth_empty_SUITE,init_per_suite, {failed,"Test failure"}}}}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}}, - + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_auto_skip,'_'},[]]}}, + {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite, {failed, {ct_cth_empty_SUITE, init_per_suite, {failed, "Test failure"}}}}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,'_'},[]]}}, + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,end_per_suite,{tc_auto_skip,'_'},[]]}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth, {'_',terminate,[[]]}}, @@ -754,11 +800,11 @@ test_events(skip_pre_suite_cth) -> {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist',{skip,"Test skip"},[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, {?eh,cth,{'_',on_tc_skip, - [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}}, {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, {?eh,test_done,{'DEF','STOP_TIME'}}, @@ -772,31 +818,33 @@ test_events(skip_pre_end_cth) -> {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,init_per_suite}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,init_per_suite,ok}}, - + [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}}, {?eh,cth,{'_',id,[[]]}}, {?eh,cth,{'_',init,['_',[]]}}, - {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, + {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, - {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, + {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}, {skipped,"Test skip"}}}], - {?eh,cth,{'_',on_tc_skip,[{end_per_group,group1}, - {tc_user_skip,{skipped,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE, + {end_per_group,group1}, + {tc_user_skip,"Test skip"}, []]}}, {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}}, {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite, {skipped,"Test skip"}}}, - {?eh,cth,{'_',on_tc_skip,[end_per_suite, - {tc_user_skip,{skipped,"Test skip"}}, + {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE, + end_per_suite, + {tc_user_skip,"Test skip"}, []]}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, @@ -808,24 +856,59 @@ test_events(skip_post_suite_cth) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{'_',init,['_',[]]}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, {?eh,cth,{'_',pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}}, {?eh,cth,{'_',on_tc_skip, - [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, + [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}}, {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, - {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, - + {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}}, + {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, - + {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[[]]}}, {?eh,stop_logging,[]} ]; +test_events(skip_pre_init_tc_cth) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,['_',[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [ct_cth_empty_SUITE,test_case,'$proplist', + {skip,"Skipped in pre_init_per_testcase"}, + []]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,test_case, + {skipped,"Skipped in pre_init_per_testcase"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [ct_cth_empty_SUITE,test_case, + {tc_user_skip,"Skipped in pre_init_per_testcase"}, + []]}}, + {?eh,test_stats,{0,0,{1,0}}}, + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_cth_empty_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(recover_post_suite_cth) -> Suite = ct_cth_fail_per_suite_SUITE, [ @@ -840,11 +923,11 @@ test_events(recover_post_suite_cth) -> {?eh,tc_start,{Suite,test_case}}, {?eh,cth,{'_',pre_init_per_testcase, - [test_case, not_contains([tc_status]),[]]}}, + [Suite,test_case, not_contains([tc_status]),[]]}}, {?eh,cth,{'_',post_end_per_testcase, - [test_case, contains([tc_status]),'_',[]]}}, + [Suite,test_case, contains([tc_status]),'_',[]]}}, {?eh,tc_done,{Suite,test_case,ok}}, - + {?eh,tc_start,{Suite,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [Suite,not_contains([tc_status]),[]]}}, @@ -861,7 +944,7 @@ test_events(update_config_cth) -> {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, {?eh,cth,{'_',init,['_',[]]}}, - + {?eh,tc_start,{ct_update_config_SUITE,init_per_suite}}, {?eh,cth,{'_',pre_init_per_suite, [ct_update_config_SUITE,contains([]),[]]}}, @@ -876,13 +959,15 @@ test_events(update_config_cth) -> {?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}}, {?eh,cth,{'_',pre_init_per_group, - [group1,contains( + [ct_update_config_SUITE, + group1,contains( [post_init_per_suite, init_per_suite, pre_init_per_suite]), []]}}, {?eh,cth,{'_',post_init_per_group, - [group1, + [ct_update_config_SUITE, + group1, contains( [post_init_per_suite, init_per_suite, @@ -898,7 +983,8 @@ test_events(update_config_cth) -> {?eh,tc_start,{ct_update_config_SUITE,test_case}}, {?eh,cth,{'_',pre_init_per_testcase, - [test_case,contains( + [ct_update_config_SUITE, + test_case,contains( [post_init_per_group, init_per_group, pre_init_per_group, @@ -907,7 +993,8 @@ test_events(update_config_cth) -> pre_init_per_suite]), []]}}, {?eh,cth,{'_',post_end_per_testcase, - [test_case,contains( + [ct_update_config_SUITE, + test_case,contains( [init_per_testcase, pre_init_per_testcase, post_init_per_group, @@ -921,7 +1008,8 @@ test_events(update_config_cth) -> {?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}}, {?eh,cth,{'_',pre_end_per_group, - [group1,contains( + [ct_update_config_SUITE, + group1,contains( [post_init_per_group, init_per_group, pre_init_per_group, @@ -930,7 +1018,8 @@ test_events(update_config_cth) -> pre_init_per_suite]), []]}}, {?eh,cth,{'_',post_end_per_group, - [group1, + [ct_update_config_SUITE, + group1, contains( [pre_end_per_group, post_init_per_group, @@ -941,7 +1030,7 @@ test_events(update_config_cth) -> pre_init_per_suite]), ok,[]]}}, {?eh,tc_done,{ct_update_config_SUITE,{end_per_group,group1,[]},ok}}, - + {?eh,tc_start,{ct_update_config_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite, [ct_update_config_SUITE,contains( @@ -974,7 +1063,7 @@ test_events(state_update_cth) -> {?eh,cth,{'_',init,['_',[]]}}, {?eh,cth,{'_',init,['_',[]]}}, {?eh,tc_start,{'_',init_per_suite}}, - + {?eh,tc_done,{'_',end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,cth,{'_',terminate,[contains( @@ -1018,10 +1107,10 @@ test_events(options_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}}, - {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}}, + {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[test]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[test]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, [ct_cth_empty_SUITE,'$proplist',[test]]}}, @@ -1051,14 +1140,14 @@ test_events(same_id_cth) -> {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}}, {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, {negative, - {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}}, + {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}}, {negative, - {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, + {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}}, - + {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, {?eh,cth,{'_',pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, {negative, @@ -1094,11 +1183,13 @@ test_events(fail_n_skip_with_minimal_cth) -> {?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case2,{skipped,"skip it"}}}, {?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,test_case3}}, {?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case3,{skipped,"skip it"}}}, - {?eh,cth,{empty_cth,on_tc_skip,[{test_case2,group2}, - {tc_user_skip,{skipped,"skip it"}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE, + {test_case2,group2}, + {tc_user_skip,"skip it"}, []]}}, - {?eh,cth,{empty_cth,on_tc_skip,[{test_case3,group2}, - {tc_user_skip,{skipped,"skip it"}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE, + {test_case3,group2}, + {tc_user_skip,"skip it"}, []]}}, {?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,{end_per_group, group2,[parallel]}}}, @@ -1115,17 +1206,25 @@ test_events(fail_n_skip_with_minimal_cth) -> ]; test_events(prio_cth) -> - - GenPre = fun(Func,States) -> - [{?eh,cth,{'_',Func,['_','_',State]}} || + GenPre = fun(Func,States) when Func==pre_init_per_suite; + Func==pre_end_per_suite -> + [{?eh,cth,{'_',Func,['_','_',State]}} || + State <- States]; + (Func,States) -> + [{?eh,cth,{'_',Func,['_','_','_',State]}} || State <- States] end, - GenPost = fun(Func,States) -> - [{?eh,cth,{'_',Func,['_','_','_',State]}} || + GenPost = fun(Func,States) when Func==post_init_per_suite; + Func==post_end_per_suite -> + [{?eh,cth,{'_',Func,['_','_','_',State]}} || + State <- States]; + (Func,States) -> + [{?eh,cth,{'_',Func,['_','_','_','_',State]}} || State <- States] - end, - + + end, + [{?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}] ++ @@ -1136,7 +1235,7 @@ test_events(prio_cth) -> [[1100,100],[600,200],[600,600],[700],[800],[900],[1000], [1200,1050],[1100],[1200]]) ++ [{?eh,tc_done,{ct_cth_prio_SUITE,init_per_suite,ok}}, - + [{?eh,tc_start,{ct_cth_prio_SUITE,{init_per_group,'_',[]}}}] ++ GenPre(pre_init_per_group, @@ -1147,7 +1246,7 @@ test_events(prio_cth) -> [900],[900,900],[500,900],[1000],[1200,1050], [1100],[1200]]) ++ [{?eh,tc_done,{ct_cth_prio_SUITE,{init_per_group,'_',[]},ok}}] ++ - + [{?eh,tc_start,{ct_cth_prio_SUITE,test_case}}] ++ GenPre(pre_init_per_testcase, [[1100,100],[600,200],[600,600],[600],[700],[800], @@ -1161,7 +1260,7 @@ test_events(prio_cth) -> [{?eh,tc_done,{ct_cth_prio_SUITE,test_case,ok}}, {?eh,tc_start,{ct_cth_prio_SUITE,{end_per_group,'_',[]}}}] ++ - GenPre(pre_end_per_group, + GenPre(pre_end_per_group, lists:reverse( [[1100,100],[600,200],[600,600],[600],[700],[800], [900],[900,900],[500,900],[1000],[1200,1050], @@ -1200,30 +1299,30 @@ test_events(no_config) -> {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_start,{ct_no_config_SUITE,test_case_1}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_1,'$proplist',[]]}}, + [ct_no_config_SUITE,test_case_1,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_1,'$proplist',ok,[]]}}, + [ct_no_config_SUITE,test_case_1,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_no_config_SUITE,test_case_1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_init_per_group, - [test_group,'$proplist',[]]}}, + [ct_no_config_SUITE,test_group,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_init_per_group, - [test_group,'$proplist','$proplist',[]]}}, + [ct_no_config_SUITE,test_group,'$proplist','$proplist',[]]}}, {?eh,tc_done,{ct_framework, {init_per_group,test_group,'$proplist'},ok}}, {?eh,tc_start,{ct_no_config_SUITE,test_case_2}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_2,'$proplist',[]]}}, + [ct_no_config_SUITE,test_case_2,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_2,'$proplist',ok,[]]}}, + [ct_no_config_SUITE,test_case_2,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_no_config_SUITE,test_case_2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_end_per_group, - [test_group,'$proplist',[]]}}, + [ct_no_config_SUITE,test_group,'$proplist',[]]}}, {?eh,cth,{empty_cth,post_end_per_group, - [test_group,'$proplist',ok,[]]}}, + [ct_no_config_SUITE,test_group,'$proplist',ok,[]]}}, {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, @@ -1236,6 +1335,166 @@ test_events(no_config) -> {?eh,stop_logging,[]} ]; +test_events(no_init_suite_config) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_no_init_suite_config_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_no_init_suite_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_no_init_suite_config_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_init_suite_config_SUITE,init_per_suite, + {failed,{error,{undef,'_'}}}}}, + {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_suite_config_SUITE, + init_per_suite, + {undef,'_'},[]]}}, + {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,test_case, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [ct_no_init_suite_config_SUITE, + test_case, + {tc_auto_skip, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}, + []]}}, + {?eh,test_stats,{0,0,{0,1}}}, + {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,end_per_suite, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [ct_no_init_suite_config_SUITE, + end_per_suite, + {tc_auto_skip, + {failed,{ct_no_init_suite_config_SUITE,init_per_suite, + {'EXIT',{undef,'_'}}}}}, + []]}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(no_init_config) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_no_init_config_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_no_init_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_no_init_config_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_no_init_config_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_no_init_config_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [ct_no_init_config_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,test_case_1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + [{?eh,tc_start,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [ct_no_init_config_SUITE,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [ct_no_init_config_SUITE,test_group,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}, + {failed,{error,{undef,'_'}}}}}, + {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_config_SUITE, + {init_per_group,test_group}, + {undef,'_'},[]]}}, + {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{test_case_2,test_group}, + {failed,{ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE, + {test_case_2,test_group}, + {tc_auto_skip, + {failed, + {ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}, + []]}}, + {?eh,test_stats,{1,0,{0,1}}}, + {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{end_per_group,test_group}, + {failed,{ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}}, + {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE, + {end_per_group,test_group}, + {tc_auto_skip, + {failed, + {ct_no_init_config_SUITE,init_per_group, + {'EXIT',{undef,'_'}}}}}, + []]}}], + {?eh,tc_start,{ct_no_init_config_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_no_init_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_no_init_config_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_init_config_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(no_end_config) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_no_end_config_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [ct_no_end_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [ct_no_end_config_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,init_per_suite,ok}}, + {?eh,tc_start,{ct_no_end_config_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_no_end_config_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [ct_no_end_config_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,test_case_1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + [{?eh,tc_start,{ct_no_end_config_SUITE, + {init_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE, + {init_per_group,test_group,'$proplist'},ok}}, + {?eh,tc_start,{ct_no_end_config_SUITE,test_case_2}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [ct_no_end_config_SUITE,test_case_2,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [ct_no_end_config_SUITE,test_case_2,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,test_case_2,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{ct_no_end_config_SUITE, + {end_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_end_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_group, + [ct_no_end_config_SUITE,test_group,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,{end_per_group,test_group,[]}, + {failed,{error,{undef,'_'}}}}}, + {?eh,cth,{empty_cth,on_tc_fail,[ct_no_end_config_SUITE, + {end_per_group,test_group}, + {undef,'_'},[]]}}], + {?eh,tc_start,{ct_no_end_config_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [ct_no_end_config_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [ct_no_end_config_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{ct_no_end_config_SUITE,end_per_suite, + {failed,{error,{undef,'_'}}}}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + test_events(data_dir) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -1250,30 +1509,30 @@ test_events(data_dir) -> {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_start,{ct_data_dir_SUITE,test_case_1}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_data_dir_SUITE,test_case_1,ok}}, {?eh,test_stats,{1,0,{0,0}}}, [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_init_per_group, - [test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_init_per_group, - [test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_framework, {init_per_group,test_group,'$proplist'},ok}}, {?eh,tc_start,{ct_data_dir_SUITE,test_case_2}}, {?eh,cth,{empty_cth,pre_init_per_testcase, - [test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_end_per_testcase, - [test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_data_dir_SUITE,test_case_2,ok}}, {?eh,test_stats,{2,0,{0,0}}}, {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, {?eh,cth,{empty_cth,pre_end_per_group, - [test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,cth,{empty_cth,post_end_per_group, - [test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, + [ct_data_dir_SUITE,test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,cth,{empty_cth,pre_end_per_suite, @@ -1300,16 +1559,654 @@ test_events(cth_log) -> [{suite,cth_log_SUITE},parallel]}}}, {?eh,tc_done,{ct_framework,{end_per_group,g1, [{suite,cth_log_SUITE},parallel]},ok}}]}, - + {?eh,tc_done,{cth_log_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]; +test_events(fallback) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [all_hook_callbacks_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [all_hook_callbacks_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [fallback_nosuite,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [fallback_nosuite,test_group,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework, + {init_per_group,test_group,'$proplist'},ok}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [fallback_nosuite,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [fallback_nosuite,test_case,'$proplist',ok,[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, + {?eh,cth,{empty_cth,pre_end_per_group, + [fallback_nosuite,test_group,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_group, + [fallback_nosuite,test_group,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], + {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [fallback_nosuite,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [fallback_nosuite,test_case,'$proplist','_',[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [fallback_nosuite,test_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [fallback_nosuite,test_case,'$proplist','_',[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}}, + {?eh,test_stats,{2,0,{0,0}}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,skip_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [fallback_nosuite,skip_case,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [fallback_nosuite,skip_case,'$proplist', + {skip,"Skipped in init_per_testcase/2"},[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,skip_case, + {skipped,"Skipped in init_per_testcase/2"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [fallback_nosuite,skip_case, + {tc_user_skip,"Skipped in init_per_testcase/2"}, + []]}}, + {?eh,test_stats,{2,0,{1,0}}}, + {?eh,tc_start,{all_hook_callbacks_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [all_hook_callbacks_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [all_hook_callbacks_SUITE,'$proplist','_',[]]}}, + {?eh,tc_done,{all_hook_callbacks_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ]; + +test_events(callbacks_on_skip) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events = + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{6,6,15}}, + + %% all_hook_callbacks_SUITE is skipped in spec + %% Only the on_tc_skip callback shall be called + {?eh,tc_user_skip,{all_hook_callbacks_SUITE,all,"Skipped in spec"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [all_hook_callbacks_SUITE,all, + {tc_user_skip,"Skipped in spec"}, + []]}}, + {?eh,test_stats,{0,0,{1,0}}}, + + %% skip_init_SUITE is skipped in its init_per_suite function + %% No group- or testcase-functions shall be called. + {?eh,tc_start,{skip_init_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [skip_init_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [skip_init_SUITE, + '$proplist', + {skip,"Skipped in init_per_suite/1"}, + []]}}, + {?eh,tc_done,{skip_init_SUITE,init_per_suite, + {skipped,"Skipped in init_per_suite/1"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_init_SUITE,init_per_suite, + {tc_user_skip,"Skipped in init_per_suite/1"}, + []]}}, + {?eh,tc_user_skip,{skip_init_SUITE,test_case,"Skipped in init_per_suite/1"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_init_SUITE,test_case, + {tc_user_skip,"Skipped in init_per_suite/1"}, + []]}}, + {?eh,test_stats,{0,0,{2,0}}}, + {?eh,tc_user_skip,{skip_init_SUITE,end_per_suite, + "Skipped in init_per_suite/1"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_init_SUITE,end_per_suite, + {tc_user_skip,"Skipped in init_per_suite/1"}, + []]}}, + + %% skip_req_SUITE is auto-skipped since a 'require' statement + %% returned by suite/0 is not fulfilled. + %% No group- or testcase-functions shall be called. + {?eh,tc_start,{skip_req_SUITE,init_per_suite}}, + {?eh,tc_done,{skip_req_SUITE,init_per_suite, + {auto_skipped,{require_failed_in_suite0, + {not_available,whatever}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_req_SUITE,init_per_suite, + {tc_auto_skip,{require_failed_in_suite0, + {not_available,whatever}}}, + []]}}, + {?eh,tc_auto_skip,{skip_req_SUITE,test_case,{require_failed_in_suite0, + {not_available,whatever}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_req_SUITE,test_case, + {tc_auto_skip,{require_failed_in_suite0, + {not_available,whatever}}}, + []]}}, + {?eh,test_stats,{0,0,{2,1}}}, + {?eh,tc_auto_skip,{skip_req_SUITE,end_per_suite, + {require_failed_in_suite0, + {not_available,whatever}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_req_SUITE,end_per_suite, + {tc_auto_skip,{require_failed_in_suite0, + {not_available,whatever}}}, + []]}}, + + %% skip_fail_SUITE is auto-skipped since the suite/0 function + %% retuns a faluty format. + %% No group- or testcase-functions shall be called. + {?eh,tc_start,{skip_fail_SUITE,init_per_suite}}, + {?eh,tc_done,{skip_fail_SUITE,init_per_suite, + {failed,{error,{suite0_failed,bad_return_value}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_fail_SUITE,init_per_suite, + {tc_auto_skip, + {failed,{error,{suite0_failed,bad_return_value}}}}, + []]}}, + {?eh,tc_auto_skip,{skip_fail_SUITE,test_case, + {failed,{error,{suite0_failed,bad_return_value}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_fail_SUITE,test_case, + {tc_auto_skip, + {failed,{error,{suite0_failed,bad_return_value}}}}, + []]}}, + {?eh,test_stats,{0,0,{2,2}}}, + {?eh,tc_auto_skip,{skip_fail_SUITE,end_per_suite, + {failed,{error,{suite0_failed,bad_return_value}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_fail_SUITE,end_per_suite, + {tc_auto_skip, + {failed,{error,{suite0_failed,bad_return_value}}}}, + []]}}, + + %% skip_group_SUITE + {?eh,tc_start,{skip_group_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [skip_group_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [skip_group_SUITE, + '$proplist', + '_', + []]}}, + {?eh,tc_done,{skip_group_SUITE,init_per_suite,ok}}, + + %% test_group_1 - auto_skip due to require failed + [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_1,[]}}}, + {?eh,tc_done, + {skip_group_SUITE,{init_per_group,test_group_1,[]}, + {auto_skipped,{require_failed,{not_available,whatever}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {init_per_group,test_group_1}, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_1}, + {require_failed,{not_available,whatever}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {test_case,test_group_1}, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + {?eh,test_stats,{0,0,{2,3}}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_1}, + {require_failed,{not_available,whatever}}}}], + %% The following appears to be outside of the group, but + %% that's only an implementation detail in + %% ct_test_support.erl - it does not know about events from + %% test suite specific hooks and regards the group ended with + %% the above tc_auto_skip-event for end_per_group. + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {end_per_group,test_group_1}, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + + %% test_group_2 - auto_skip due to failed return from group/1 + [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_2,[]}}}, + {?eh,tc_done, + {skip_group_SUITE,{init_per_group,test_group_2,[]}, + {auto_skipped,{group0_failed,bad_return_value}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {init_per_group,test_group_2}, + {tc_auto_skip,{group0_failed,bad_return_value}}, + []]}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_2}, + {group0_failed,bad_return_value}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {test_case,test_group_2}, + {tc_auto_skip,{group0_failed,bad_return_value}}, + []]}}, + {?eh,test_stats,{0,0,{2,4}}}, + {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_2}, + {group0_failed,bad_return_value}}}], + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {end_per_group,test_group_2}, + {tc_auto_skip,{group0_failed,bad_return_value}}, + []]}}, + %% test_group_3 - user_skip in init_per_group/2 + [{?eh,tc_start, + {skip_group_SUITE,{init_per_group,test_group_3,[]}}}, + {?eh,cth,{empty_cth,pre_init_per_group, + [skip_group_SUITE,test_group_3,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_group, + [skip_group_SUITE,test_group_3,'$proplist', + {skip,"Skipped in init_per_group/2"}, + []]}}, + {?eh,tc_done,{skip_group_SUITE, + {init_per_group,test_group_3,[]}, + {skipped,"Skipped in init_per_group/2"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {init_per_group,test_group_3}, + {tc_user_skip,"Skipped in init_per_group/2"}, + []]}}, + {?eh,tc_user_skip,{skip_group_SUITE, + {test_case,test_group_3}, + "Skipped in init_per_group/2"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {test_case,test_group_3}, + {tc_user_skip,"Skipped in init_per_group/2"}, + []]}}, + {?eh,test_stats,{0,0,{3,4}}}, + {?eh,tc_user_skip,{skip_group_SUITE, + {end_per_group,test_group_3}, + "Skipped in init_per_group/2"}}], + {?eh,cth,{empty_cth,on_tc_skip, + [skip_group_SUITE, + {end_per_group,test_group_3}, + {tc_user_skip,"Skipped in init_per_group/2"}, + []]}}, + + {?eh,tc_start,{skip_group_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [skip_group_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [skip_group_SUITE, + '$proplist', + ok,[]]}}, + {?eh,tc_done,{skip_group_SUITE,end_per_suite,ok}}, + + + %% skip_case_SUITE has 4 test cases which are all skipped in + %% different ways + {?eh,tc_start,{skip_case_SUITE,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [skip_case_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [skip_case_SUITE, + '$proplist', + '_', + []]}}, + {?eh,tc_done,{skip_case_SUITE,init_per_suite,ok}}, + + %% Skip in spec -> only on_tc_skip shall be called + {?eh,tc_user_skip,{skip_case_SUITE,skip_in_spec,"Skipped in spec"}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,skip_in_spec, + {tc_user_skip,"Skipped in spec"}, + []]}}, + {?eh,test_stats,{0,0,{4,4}}}, + + %% Skip in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,skip_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,skip_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,skip_in_init, + '$proplist', + {skip,"Skipped in init_per_testcase/2"}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,skip_in_init, + {skipped,"Skipped in init_per_testcase/2"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,skip_in_init, + {tc_user_skip,"Skipped in init_per_testcase/2"}, + []]}}, + {?eh,test_stats,{0,0,{5,4}}}, + + %% Fail in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,fail_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,fail_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,fail_in_init, + '$proplist', + {skip,{failed,'_'}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,fail_in_init, + {auto_skipped,{failed,'_'}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,fail_in_init, + {tc_auto_skip,{failed,'_'}}, + []]}}, + {?eh,test_stats,{0,0,{5,5}}}, + + %% Exit in init_per_testcase -> pre/post_end_per_testcase + %% shall not be called + {?eh,tc_start,{skip_case_SUITE,exit_in_init}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,exit_in_init, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,exit_in_init, + '$proplist', + {skip,{failed,'_'}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,exit_in_init, + {auto_skipped,{failed,'_'}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,exit_in_init, + {tc_auto_skip,{failed,'_'}}, + []]}}, + {?eh,test_stats,{0,0,{5,6}}}, + + %% Fail in end_per_testcase -> all hooks shall be called and + %% test shall succeed. + {?eh,tc_start,{skip_case_SUITE,fail_in_end}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + ok, + []]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,fail_in_end, + '$proplist', + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT', + {test_case_failed,"Failed in end_per_testcase/2"}}}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,fail_in_end, + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT', + {test_case_failed,"Failed in end_per_testcase/2"}}}}}}, + {?eh,test_stats,{1,0,{5,6}}}, + + %% Exit in end_per_testcase -> all hooks shall be called and + %% test shall succeed. + {?eh,tc_start,{skip_case_SUITE,exit_in_end}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + ok, + []]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,exit_in_end, + '$proplist', + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT',"Exit in end_per_testcase/2"}}}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,exit_in_end, + {failed, + {skip_case_SUITE,end_per_testcase, + {'EXIT',"Exit in end_per_testcase/2"}}}}}, + {?eh,test_stats,{2,0,{5,6}}}, + + %% Skip in testcase function -> all callbacks shall be called + {?eh,tc_start,{skip_case_SUITE,skip_in_case}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + ok,[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [skip_case_SUITE,skip_in_case, + '$proplist', + {skip,"Skipped in test case function"}, + []]}}, + {?eh,tc_done,{skip_case_SUITE,skip_in_case, + {skipped,"Skipped in test case function"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,skip_in_case, + {tc_user_skip,"Skipped in test case function"}, + []]}}, + {?eh,test_stats,{2,0,{6,6}}}, + + %% Auto skip due to failed 'require' -> only the on_tc_skip + %% callback shall be called + {?eh,tc_start,{skip_case_SUITE,req_auto_skip}}, + {?eh,tc_done,{skip_case_SUITE,req_auto_skip, + {auto_skipped,{require_failed,{not_available,whatever}}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,req_auto_skip, + {tc_auto_skip,{require_failed,{not_available,whatever}}}, + []]}}, + {?eh,test_stats,{2,0,{6,7}}}, + + %% Auto skip due to failed testcase/0 function -> only the + %% on_tc_skip callback shall be called + {?eh,tc_start,{skip_case_SUITE,fail_auto_skip}}, + {?eh,tc_done,{skip_case_SUITE,fail_auto_skip, + {auto_skipped,{testcase0_failed,bad_return_value}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [skip_case_SUITE,fail_auto_skip, + {tc_auto_skip,{testcase0_failed,bad_return_value}}, + []]}}, + {?eh,test_stats,{2,0,{6,8}}}, + + {?eh,tc_start,{skip_case_SUITE,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [skip_case_SUITE, + '$proplist', + []]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [skip_case_SUITE, + '$proplist', + ok,[]]}}, + {?eh,tc_done,{skip_case_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(failed_sequence) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events = + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[seq_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [seq_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{seq_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [seq_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [seq_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [seq_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [seq_SUITE,test_case_1,'$proplist', + {error,failed_on_purpose},[]]}}, + {?eh,tc_done,{seq_SUITE,test_case_1,{failed,{error,failed_on_purpose}}}}, + {?eh,cth,{empty_cth,on_tc_fail, + [seq_SUITE,test_case_1,failed_on_purpose,[]]}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{seq_SUITE,test_case_2}}, + {?eh,tc_done,{seq_SUITE,test_case_2, + {auto_skipped,{sequence_failed,seq1,test_case_1}}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [seq_SUITE,test_case_2, + {tc_auto_skip,{sequence_failed,seq1,test_case_1}}, + []]}}, + {?eh,test_stats,{0,1,{0,1}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[seq_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite,[seq_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(repeat_force_stop) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events= + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,2}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite,[repeat_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [repeat_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{repeat_SUITE,test_case_1}}, + {?eh,cth,{empty_cth,pre_init_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,cth,{empty_cth,pre_end_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_testcase, + [repeat_SUITE,test_case_1,'$proplist',ok,[]]}}, + {?eh,tc_done,{repeat_SUITE,test_case_1,ok}}, + {?eh,test_stats,{1,0,{0,0}}}, + {?eh,tc_start,{repeat_SUITE,test_case_2}}, + {?eh,tc_done,{repeat_SUITE,test_case_2, + {auto_skipped, + "Repeated test stopped by force_stop option"}}}, + {?eh,cth,{empty_cth,on_tc_skip, + [repeat_SUITE,test_case_2, + {tc_auto_skip,"Repeated test stopped by force_stop option"}, + []]}}, + {?eh,test_stats,{1,0,{0,1}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite,[repeat_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [repeat_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(config_clash) -> + %% skip_cth.erl will send a 'cth_error' event if a hook is + %% erroneously called. Therefore, all Events are changed to + %% {negative,{?eh,cth_error,'_'},Event} + %% at the end of this function. + Events = + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,cth,{empty_cth,id,[[]]}}, + {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, + {?eh,start_info,{1,1,1}}, + {?eh,tc_start,{ct_framework,init_per_suite}}, + {?eh,cth,{empty_cth,pre_init_per_suite, + [config_clash_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_init_per_suite, + [config_clash_SUITE,'$proplist','$proplist',[]]}}, + {?eh,tc_done,{ct_framework,init_per_suite,ok}}, + {?eh,tc_start,{config_clash_SUITE,test_case_1}}, + {?eh,tc_done,{config_clash_SUITE,test_case_1, + {failed,{error,{config_name_already_in_use,[aa]}}}}}, + {?eh,cth,{empty_cth,on_tc_fail, + [config_clash_SUITE,test_case_1, + {config_name_already_in_use,[aa]}, + []]}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{ct_framework,end_per_suite}}, + {?eh,cth,{empty_cth,pre_end_per_suite, + [config_clash_SUITE,'$proplist',[]]}}, + {?eh,cth,{empty_cth,post_end_per_suite, + [config_clash_SUITE,'$proplist',ok,[]]}}, + {?eh,tc_done,{ct_framework,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,cth,{empty_cth,terminate,[[]]}}, + {?eh,stop_logging,[]} + ], + %% Make sure no 'cth_error' events are received! + [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + test_events(ok) -> ok. - %% test events help functions contains(List) -> fun(Proplist) when is_list(Proplist) -> diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl new file mode 100644 index 0000000000..af8a85cd75 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl @@ -0,0 +1,62 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(all_hook_callbacks_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(Config) -> + Config. + +end_per_group(_Config) -> + ok. + +init_per_testcase(skip_case, Config) -> + {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +all() -> + [{group,test_group},test_case,skip_case]. + +groups() -> + [{test_group,[test_case]}]. + +%% Test cases starts here. +test_case(Config) -> + ok. + +skip_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl new file mode 100644 index 0000000000..3853dc29da --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl @@ -0,0 +1,43 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(config_clash_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + [{require,aa,yy},{default_config,yy,"this is a default value"}]. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case_1]. + +%% Test cases starts here. +test_case_1() -> + [{require,aa,xx}]. +test_case_1(_Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl new file mode 100644 index 0000000000..0b6cabead3 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl @@ -0,0 +1,51 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_end_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that all pre/post_end_per_* callbacks +%%% are called with correct SuiteName even if no end_per_* config +%%% function exist in the suite, and that the non-exported config +%%% functions fail with 'undef'. + +init_per_suite(Config) -> + Config. + +init_per_group(_Group,Config) -> + Config. + +init_per_testcase(_TC,Config) -> + Config. + +all() -> + [test_case_1, {group,test_group}]. + +groups() -> + [{test_group,[],[test_case_2]}]. + +test_case_1(Config) -> + ok. + +test_case_2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl new file mode 100644 index 0000000000..29cdc21ee4 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl @@ -0,0 +1,54 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_init_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that all +%%% pre/post_init_per_group/testcase callbacks are called with correct +%%% SuiteName even if no init_per_group/testcase function exist in the +%%% suite, and that the non-exported config functions fail with 'undef'. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +end_per_group(_Group,Config) -> + Config. + +end_per_testcase(_TC,Config) -> + Config. + +all() -> + [test_case_1, {group,test_group}]. + +groups() -> + [{test_group,[],[test_case_2]}]. + +test_case_1(Config) -> + ok. + +test_case_2(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl new file mode 100644 index 0000000000..d6daff9792 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl @@ -0,0 +1,39 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_init_suite_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that pre/post_init_per_suite +%%% callbacks are called with correct SuiteName even if no +%%% init_per_suite function exist in the suite, and that the +%%% non-exported config function fails with 'undef'. + +end_per_suite(Config) -> + Config. + +all() -> + [test_case]. + +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl index c00eb5cf93..961ea68d2d 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,18 +44,18 @@ -export([pre_end_per_suite/3]). -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([pre_end_per_group/4]). +-export([post_end_per_group/5]). --export([pre_init_per_testcase/3]). --export([post_init_per_testcase/4]). --export([pre_end_per_testcase/3]). --export([post_end_per_testcase/4]). +-export([pre_init_per_testcase/4]). +-export([post_init_per_testcase/5]). +-export([pre_end_per_testcase/4]). +-export([post_end_per_testcase/5]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). +-export([on_tc_fail/4]). +-export([on_tc_skip/4]). -export([terminate/1]). @@ -154,150 +154,160 @@ post_end_per_suite(Suite,Config,Return,State) -> %% @doc Called before each init_per_group. %% You can change the config in this function. --spec pre_init_per_group(Group :: atom(), - Config :: config(), - State :: #state{}) -> +-spec pre_init_per_group(Suite :: atom(), + Group :: atom(), + Config :: config(), + State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_init_per_group, - [Group,Config,State]}}), - ct:log("~w:pre_init_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,State]}}), + ct:log("~w:pre_init_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Config, State}. %% @doc Called after each init_per_group. %% You can change the return value in this function. --spec post_init_per_group(Group :: atom(), +-spec post_init_per_group(Suite :: atom(), + Group :: atom(), Config :: config(), Return :: config() | skip_or_fail(), State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_init_per_group, - [Group,Config,Return,State]}}), - ct:log("~w:post_init_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,Return,State]}}), + ct:log("~w:post_init_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Return, State}. %% @doc Called after each end_per_group. The config/state can be changed here, %% though it will only affect the *end_per_group functions. --spec pre_end_per_group(Group :: atom(), +-spec pre_end_per_group(Suite :: atom(), + Group :: atom(), Config :: config() | skip_or_fail(), State :: #state{}) -> {ok | skip_or_fail(), NewState :: #state{}}. -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_end_per_group, - [Group,Config,State]}}), - ct:log("~w:pre_end_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,State]}}), + ct:log("~w:pre_end_per_group(~w~w) called", [?MODULE,Suite,Group]), {Config, State}. %% @doc Called after each end_per_group. Note that the config cannot be %% changed here, only the status of the group. --spec post_end_per_group(Group :: atom(), +-spec post_end_per_group(Suite :: atom(), + Group :: atom(), Config :: config(), Return :: term(), State :: #state{}) -> {ok | skip_or_fail(), NewState :: #state{}}. -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_end_per_group, - [Group,Config,Return,State]}}), - ct:log("~w:post_end_per_group(~w) called", [?MODULE,Group]), + [Suite,Group,Config,Return,State]}}), + ct:log("~w:post_end_per_group(~w,~w) called", [?MODULE,Suite,Group]), {Return, State}. %% @doc Called before init_per_testcase/2 for each test case. %% You can change the config in this function. --spec pre_init_per_testcase(TC :: atom(), - Config :: config(), - State :: #state{}) -> +-spec pre_init_per_testcase(Suite :: atom(), + TC :: atom(), + Config :: config(), + State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_init_per_testcase, - [TC,Config,State]}}), - ct:log("~w:pre_init_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,State]}}), + ct:log("~w:pre_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Config, State}. %% @doc Called after init_per_testcase/2, and before the test case. --spec post_init_per_testcase(TC :: atom(), +-spec post_init_per_testcase(Suite :: atom(), + TC :: atom(), Config :: config(), Return :: config() | skip_or_fail(), State :: #state{}) -> {config() | skip_or_fail(), NewState :: #state{}}. -post_init_per_testcase(TC,Config,Return,State) -> +post_init_per_testcase(Suite,TC,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_init_per_testcase, - [TC,Config,Return,State]}}), - ct:log("~w:post_init_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,Return,State]}}), + ct:log("~w:post_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Return, State}. %% @doc Called before end_per_testacse/2. No skip or fail allowed here, %% only config additions. --spec pre_end_per_testcase(TC :: atom(), - Config :: config(), - State :: #state{}) -> +-spec pre_end_per_testcase(Suite :: atom(), + TC :: atom(), + Config :: config(), + State :: #state{}) -> {config(), NewState :: #state{}}. -pre_end_per_testcase(TC,Config,State) -> +pre_end_per_testcase(Suite,TC,Config,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, pre_end_per_testcase, - [TC,Config,State]}}), - ct:log("~w:pre_end_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,State]}}), + ct:log("~w:pre_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Config, State}. %% @doc Called after end_per_testcase/2 for each test case. Note that %% the config cannot be changed here, only the status of the test case. --spec post_end_per_testcase(TC :: atom(), +-spec post_end_per_testcase(Suite :: atom(), + TC :: atom(), Config :: config(), Return :: term(), State :: #state{}) -> {ok | skip_or_fail(), NewState :: #state{}}. -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, post_end_per_testcase, - [TC,Config,Return,State]}}), - ct:log("~w:post_end_per_testcase(~w) called", [?MODULE,TC]), + [Suite,TC,Config,Return,State]}}), + ct:log("~w:post_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]), {Return, State}. %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group, %% post_end_per_group and post_end_per_tc if the suite, group or test case failed. %% This function should be used for extra cleanup which might be needed. %% It is not possible to modify the config or the status of the test run. --spec on_tc_fail(TC :: init_per_suite | end_per_suite | +-spec on_tc_fail(Suite :: atom(), + TC :: init_per_suite | end_per_suite | init_per_group | end_per_group | atom() | {Function :: atom(), GroupName :: atom()}, Reason :: term(), State :: #state{}) -> NewState :: #state{}. -on_tc_fail(TC, Reason, State) -> +on_tc_fail(Suite, TC, Reason, State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, on_tc_fail, - [TC,Reason,State]}}), - ct:log("~w:on_tc_fail(~w) called", [?MODULE,TC]), + [Suite,TC,Reason,State]}}), + ct:log("~w:on_tc_fail(~w,~w) called", [?MODULE,Suite,TC]), State. %% @doc Called when a test case is skipped by either user action %% or due to an init function failing. Test case can be %% end_per_suite, init_per_group, end_per_group and the actual test cases. --spec on_tc_skip(TC :: end_per_suite | +-spec on_tc_skip(Suite :: atom(), + TC :: end_per_suite | init_per_group | end_per_group | atom() | {Function :: atom(), GroupName :: atom()}, {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} | {tc_user_skip, {skipped, Reason :: term()}}, State :: #state{}) -> NewState :: #state{}. -on_tc_skip(TC, Reason, State) -> +on_tc_skip(Suite, TC, Reason, State) -> gen_event:notify( ?CT_EVMGR_REF, #event{ name = cth, node = node(), data = {?MODULE, on_tc_skip, - [TC,Reason,State]}}), - ct:log("~w:on_tc_skip(~w) called", [?MODULE,TC]), + [Suite,TC,Reason,State]}}), + ct:log("~w:on_tc_skip(~w,~w) called", [?MODULE,Suite,TC]), State. %% @doc Called when the scope of the CTH is done, this depends on diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl index 559b22bc9f..7cb2d19414 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl @@ -1,7 +1,7 @@ %%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
%%
%% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,29 +45,29 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) ->
empty_cth:post_end_per_suite(Suite,Config,Return,State).
-pre_init_per_group(Group,Config,State) ->
- empty_cth:pre_init_per_group(Group,Config,State).
+pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) ->
- empty_cth:post_init_per_group(Group,Config,Return,State).
+post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) ->
- empty_cth:pre_end_per_group(Group,Config,State).
+pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) ->
- empty_cth:post_end_per_group(Group,Config,Return,State).
+post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) ->
- empty_cth:pre_init_per_testcase(TC,Config,State).
+pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) ->
- empty_cth:post_end_per_testcase(TC,Config,Return,State).
+post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) ->
- empty_cth:on_tc_fail(TC,Reason,State).
+on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) ->
- empty_cth:on_tc_skip(TC,Reason,State).
+on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) ->
empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl index 51202443bf..ebe6014127 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl new file mode 100644 index 0000000000..665efd659a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl @@ -0,0 +1,81 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(fallback_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Group,Config,State) -> + empty_cth:pre_init_per_group(fallback_nosuite,Group,Config,State). + +post_init_per_group(Group,Config,Return,State) -> + empty_cth:post_init_per_group(fallback_nosuite,Group,Config,Return,State). + +pre_end_per_group(Group,Config,State) -> + empty_cth:pre_end_per_group(fallback_nosuite,Group,Config,State). + +post_end_per_group(Group,Config,Return,State) -> + empty_cth:post_end_per_group(fallback_nosuite,Group,Config,Return,State). + +pre_init_per_testcase(TC,Config,State) -> + empty_cth:pre_init_per_testcase(fallback_nosuite,TC,Config,State). + +post_init_per_testcase(TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(fallback_nosuite,TC,Config,Return,State). + +pre_end_per_testcase(TC,Config,State) -> + empty_cth:pre_end_per_testcase(fallback_nosuite,TC,Config,State). + +post_end_per_testcase(TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(fallback_nosuite,TC,Config,Return,State). + +on_tc_fail(TC, Reason, State) -> + empty_cth:on_tc_fail(fallback_nosuite,TC,Reason,State). + +on_tc_skip(TC, Reason, State) -> + empty_cth:on_tc_skip(fallback_nosuite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl index b49cbe7fb4..f5d215c69d 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -29,13 +29,13 @@ %% CT Hooks -export([init/2]). -export([terminate/1]). --export([on_tc_skip/3]). +-export([on_tc_skip/4]). init(Id, Opts) -> empty_cth:init(Id, Opts). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite, TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl index a687743641..d462e5e071 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl index 4d9c60f1ca..f67d152cf1 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl new file mode 100644 index 0000000000..a1849839c9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl @@ -0,0 +1,42 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(repeat_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case_1, test_case_2]. + +%% Test cases starts here. +test_case_1(_Config) -> + timer:sleep(10000), + ok. + +test_case_2(_Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl index 494f398fc1..a4105f5829 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -48,35 +48,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl new file mode 100644 index 0000000000..30eebb272a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl @@ -0,0 +1,45 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(seq_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [{sequence,seq1}]. + +sequences() -> + [{seq1,[test_case_1,test_case_2]}]. + +%% Test cases starts here. +test_case_1(_Config) -> + exit(failed_on_purpose). + +test_case_2(_Config) -> + ct:fail("This test shall never be run since test_case_1 fails " + "and they are run in sequence"). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec new file mode 100644 index 0000000000..a271c5e8b2 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec @@ -0,0 +1,8 @@ +{suites,".",[all_hook_callbacks_SUITE, + skip_init_SUITE, + skip_req_SUITE, + skip_fail_SUITE, + skip_group_SUITE, + skip_case_SUITE]}. +{skip_suites,".",all_hook_callbacks_SUITE,"Skipped in spec"}. +{skip_cases,".",skip_case_SUITE,skip_in_spec,"Skipped in spec"}. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl new file mode 100644 index 0000000000..3c2275d55f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl @@ -0,0 +1,106 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_case_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(skip_in_init,Config) -> + {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(fail_in_init,Config) -> + ct:fail("Failed in init_per_testcase/2"); +init_per_testcase(exit_in_init,Config) -> + exit(self(),"Exit in init_per_testcase/2"); +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(fail_in_end,_) -> + ct:fail("Failed in end_per_testcase/2"); +end_per_testcase(exit_in_end,_) -> + exit(self(),"Exit in end_per_testcase/2"); +end_per_testcase(_,_) -> + ok. + +all() -> + [skip_in_spec, + skip_in_init, + fail_in_init, + exit_in_init, + fail_in_end, + exit_in_end, + skip_in_case, + req_auto_skip, + fail_auto_skip + ]. + +%% Test cases starts here. +skip_in_spec(Config) -> + ct:fail("This test shall never be run. " + "It shall be skipped in the test spec."). + +skip_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall be skipped in init_per_testcase/2."). + +fail_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall fail in init_per_testcase/2."). + +exit_in_init(Config) -> + ct:fail("This test shall never be run. " + "It shall exit in init_per_testcase/2."). + +fail_in_end(Config) -> + ok. + +exit_in_end(Config) -> + ok. + +skip_in_case(Config) -> + {skip,"Skipped in test case function"}. + +req_auto_skip() -> + [{require,whatever}]. +req_auto_skip(Config) -> + ct:fail("This test shall never be run due to " + "failed require"). + +fail_auto_skip() -> + faulty_return_value. +fail_auto_skip(Config) -> + ct:fail("This test shall never be run due to " + "faulty return from info function"). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl new file mode 100644 index 0000000000..7b5e93f836 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl @@ -0,0 +1,182 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(skip_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments +-define(fail(Info), + gen_event:notify( + ?CT_EVMGR_REF, + #event{ name = cth_error, + node = node(), + data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})). + +%% CT Hooks +-compile(export_all). + +id(Opts) -> + empty_cth:id(Opts). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + Suite==skip_init_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==skip_case_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + Suite==skip_init_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==skip_case_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + Suite==skip_case_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + Suite==skip_case_SUITE + orelse Suite==skip_group_SUITE + orelse Suite==seq_SUITE + orelse Suite==repeat_SUITE + orelse Suite==config_clash_SUITE + orelse ?fail(Suite), + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + (Suite==skip_group_SUITE andalso Group==test_group_3) + orelse ?fail({Suite,Group}), + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + (Suite==skip_group_SUITE andalso Group==test_group_3) + orelse ?fail({Suite,Group}), + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + ?fail({Suite,Group}), + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + ?fail({Suite,Group}), + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_init + orelse TC==fail_in_init + orelse TC==exit_in_init + orelse TC==fail_in_end + orelse TC==exit_in_end + orelse TC==skip_in_case)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_init + orelse TC==fail_in_init + orelse TC==exit_in_init + orelse TC==fail_in_end + orelse TC==exit_in_end + orelse TC==skip_in_case)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_case + orelse TC==fail_in_end + orelse TC==exit_in_end)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + (Suite==skip_case_SUITE andalso (TC==skip_in_case + orelse TC==fail_in_end + orelse TC==exit_in_end)) + orelse (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==repeat_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC,Reason,State) -> + (Suite==seq_SUITE andalso TC==test_case_1) + orelse (Suite==config_clash_SUITE andalso TC==test_case_1) + orelse ?fail({Suite,TC}), + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(all_hook_callbacks_SUITE=Suite,all=TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) + when (Suite==skip_init_SUITE + orelse Suite==skip_req_SUITE + orelse Suite==skip_fail_SUITE) + andalso + (TC==init_per_suite + orelse TC==test_case + orelse TC==end_per_suite) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(skip_group_SUITE=Suite,TC={C,G},Reason,State) + when (C==init_per_group orelse C==test_case orelse C==end_per_group) andalso + (G==test_group_1 orelse G==test_group_2 orelse G==test_group_3) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State) + when TC==skip_in_spec; + TC==skip_in_init; + TC==fail_in_init; + TC==exit_in_init; + TC==skip_in_case; + TC==req_auto_skip; + TC==fail_auto_skip -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) + when (Suite==seq_SUITE andalso TC==test_case_2) + orelse (Suite==repeat_SUITE andalso TC==test_case_2) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) -> + ?fail({Suite,TC}), + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl new file mode 100644 index 0000000000..df8835598e --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_fail_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + faulty_return_value. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl new file mode 100644 index 0000000000..8d25701a22 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl @@ -0,0 +1,64 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_group_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +group(test_group_1) -> + [{require,whatever}]; +group(test_group_2) -> + faulty_return_value; +group(_) -> + []. + +init_per_group(test_group_3,Config) -> + {skip,"Skipped in init_per_group/2"}; +init_per_group(_,Config) -> + ct:fail("This shall never be run due to auto_skip from group/1"). + +end_per_group(_,_) -> + ct:fail("This shall never be run"). + +all() -> + [{group,test_group_1}, + {group,test_group_2}, + {group,test_group_3}]. + +groups() -> + [{test_group_1,[test_case]}, + {test_group_2,[test_case]}, + {test_group_3,[test_case]}]. + +%% Test cases starts here. +test_case(_Config) -> + ct:fail("This test case shall never be run due to skip on group level"). + diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl new file mode 100644 index 0000000000..7d01ff58a9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_init_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + []. + +init_per_suite(Config) -> + {skip,"Skipped in init_per_suite/1"}. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl index d5b347e723..b3b5db23fb 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl index 36abac0bf8..525c376830 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -46,36 +46,36 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State), {{skip, "Test skip"}, State}. -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl new file mode 100644 index 0000000000..459608ac07 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl @@ -0,0 +1,79 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(skip_pre_init_tc_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +init(Id, Opts) -> + empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> + empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> + empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> + empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> + empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State), + {{skip, "Skipped in pre_init_per_testcase"}, State}. + +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> + empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl index fa510b2d54..1e6cab04ed 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -46,35 +46,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl new file mode 100644 index 0000000000..c6e67d978e --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_req_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> + [{require,whatever}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + ok. + +init_per_group(_,Config) -> + Config. + +end_per_group(_,_) -> + ok. + +init_per_testcase(_,Config) -> + Config. + +end_per_testcase(_,_) -> + ok. + +all() -> + [test_case]. + +%% Test cases starts here. +test_case(Config) -> + ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl index 7ec0d458b6..2b37116cb2 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -48,44 +48,44 @@ post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State), {Return, [post_end_per_suite|State]}. -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State), +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State), {Config, [pre_init_per_group|State]}. -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State), +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State), {Return, [post_init_per_group|State]}. -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State), {Config, [pre_end_per_group|State]}. -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State), +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State), {Return, [post_end_per_group|State]}. -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State), +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State), {Config, [pre_init_per_testcase|State]}. -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State), +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State), {Return, [post_init_per_testcase|State]}. -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State), +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State), {Config, [pre_end_per_testcase|State]}. -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State), +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State), {Return, [post_end_per_testcase|State]}. -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State), +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State), [on_tc_fail|State]. -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State), +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State), [on_tc_skip|State]. terminate(State) -> diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl index 2b9e726819..ac7b74539a 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -44,35 +44,35 @@ pre_end_per_suite(Suite,Config,State) -> post_end_per_suite(Suite,Config,Return,State) -> empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl index d48981f667..7b0c1f599f 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -50,43 +50,43 @@ post_end_per_suite(Suite,Config,Return,State) -> NewConfig = [{post_end_per_suite,?now}|Config], {NewConfig,NewConfig}. -pre_init_per_group(Group,Config,State) -> - empty_cth:pre_init_per_group(Group,Config,State), +pre_init_per_group(Suite, Group,Config,State) -> + empty_cth:pre_init_per_group(Suite,Group,Config,State), {[{pre_init_per_group,?now}|Config],State}. -post_init_per_group(Group,Config,Return,State) -> - empty_cth:post_init_per_group(Group,Config,Return,State), +post_init_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_init_per_group(Suite,Group,Config,Return,State), {[{post_init_per_group,?now}|Return],State}. -pre_end_per_group(Group,Config,State) -> - empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> + empty_cth:pre_end_per_group(Suite,Group,Config,State), {[{pre_end_per_group,?now}|Config],State}. -post_end_per_group(Group,Config,Return,State) -> - empty_cth:post_end_per_group(Group,Config,Return,State), +post_end_per_group(Suite,Group,Config,Return,State) -> + empty_cth:post_end_per_group(Suite,Group,Config,Return,State), {[{post_end_per_group,?now}|Config],State}. -pre_init_per_testcase(TC,Config,State) -> - empty_cth:pre_init_per_testcase(TC,Config,State), +pre_init_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_init_per_testcase(Suite,TC,Config,State), {[{pre_init_per_testcase,?now}|Config],State}. -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State), +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State), {[{post_init_per_testcase,?now}|Config],State}. -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State), +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State), {[{pre_end_per_testcase,?now}|Config],State}. -post_end_per_testcase(TC,Config,Return,State) -> - empty_cth:post_end_per_testcase(TC,Config,Return,State), +post_end_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State), {[{post_end_per_testcase,?now}|Config],State}. -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl index 71d84781e0..44e7c7c29d 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -60,37 +60,37 @@ post_end_per_suite(Suite,Config,Return,State) -> ct_no_config_SUITE = ct:get_config(suite_cfg), empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) -> true = ?val(post_init_per_suite, Config), ct_no_config_SUITE = ct:get_config(suite_cfg), test_group = ct:get_config(group_cfg), - empty_cth:pre_init_per_group(Group, + empty_cth:pre_init_per_group(Suite,Group, [{pre_init_per_group,true} | Config], State). -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) -> true = ?val(pre_init_per_group, Return), test_group = ct:get_config(group_cfg), - empty_cth:post_init_per_group(Group, + empty_cth:post_init_per_group(Suite,Group, Config, [{post_init_per_group,true} | Return], State). -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) -> true = ?val(post_init_per_group, Config), ct_no_config_SUITE = ct:get_config(suite_cfg), test_group = ct:get_config(group_cfg), - empty_cth:pre_end_per_group(Group, + empty_cth:pre_end_per_group(Suite,Group, [{pre_end_per_group,true} | Config], State). -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) -> true = ?val(pre_end_per_group, Config), ct_no_config_SUITE = ct:get_config(suite_cfg), test_group = ct:get_config(group_cfg), - empty_cth:post_end_per_group(Group,Config,Return,State). + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) -> true = ?val(post_init_per_suite, Config), case ?val(name, ?val(tc_group_properties, Config)) of undefined -> @@ -102,19 +102,19 @@ pre_init_per_testcase(TC,Config,State) -> ct_no_config_SUITE = ct:get_config(suite_cfg), CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"), TC = ct:get_config(CfgKey), - empty_cth:pre_init_per_testcase(TC, + empty_cth:pre_init_per_testcase(Suite,TC, [{pre_init_per_testcase,true} | Config], State). %%! TODO: Verify Config also in post_init and pre_end! -post_init_per_testcase(TC,Config,Return,State) -> - empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> - empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) -> true = ?val(post_init_per_suite, Config), true = ?val(pre_init_per_testcase, Config), case ?val(name, ?val(tc_group_properties, Config)) of @@ -127,13 +127,13 @@ post_end_per_testcase(TC,Config,Return,State) -> ct_no_config_SUITE = ct:get_config(suite_cfg), CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"), TC = ct:get_config(CfgKey), - empty_cth:post_end_per_testcase(TC,Config,Return,State). + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl index 9abd2e5e83..bde6744433 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% Copyright Ericsson AB 2010-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -62,43 +62,43 @@ post_end_per_suite(Suite,Config,Return,State) -> check_dirs(State,Config), empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) -> check_dirs(State,Config), - empty_cth:pre_init_per_group(Group,Config,State). + empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) -> check_dirs(State,Return), - empty_cth:post_init_per_group(Group,Config,Return,State). + empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) -> check_dirs(State,Config), - empty_cth:pre_end_per_group(Group,Config,State). + empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) -> check_dirs(State,Config), - empty_cth:post_end_per_group(Group,Config,Return,State). + empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) -> check_dirs(State,Config), - empty_cth:pre_init_per_testcase(TC,Config,State). + empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> +post_init_per_testcase(Suite,TC,Config,Return,State) -> check_dirs(State,Config), - empty_cth:post_init_per_testcase(TC,Config,Return,State). + empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> +pre_end_per_testcase(Suite,TC,Config,State) -> check_dirs(State,Config), - empty_cth:pre_end_per_testcase(TC,Config,State). + empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) -> check_dirs(State,Config), - empty_cth:post_end_per_testcase(TC,Config,Return,State). + empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> - empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> + empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> - empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> + empty_cth:on_tc_skip(Suite,TC,Reason,State). terminate(State) -> empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_keep_logs_SUITE.erl b/lib/common_test/test/ct_keep_logs_SUITE.erl new file mode 100644 index 0000000000..6b7aaa57ac --- /dev/null +++ b/lib/common_test/test/ct_keep_logs_SUITE.erl @@ -0,0 +1,199 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_keep_logs_SUITE +%%% +%%% Description: +%%% Test the 'keep_logs' option +%%% +%%%------------------------------------------------------------------- +-module(ct_keep_logs_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +init_per_suite(Config0) -> + ct_test_support:init_per_suite(Config0). + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + keep_logs, + refresh_logs + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%% Test the keep_logs option with normal common_test runs +keep_logs(Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "keep_logs_SUITE"), + Opts0 = ct_test_support:get_opts(Config), + Opts = [{suite,Suite},{label,keep_logs} | Opts0], + + LogDir=?config(logdir,Opts), + KeepLogsDir = create_dir(filename:join(LogDir,"keep_logs-")), + Opts1 = lists:keyreplace(logdir,1,Opts,{logdir,KeepLogsDir}), + ct:log("New LogDir = ~s", [KeepLogsDir]), + + %% Create 6 ct_run.* log directories + [ok = ct_test_support:run(Opts1, Config) || _ <- lists:seq(1,3)], + + %% Verify the number of directories + WC = filename:join(KeepLogsDir,"ct_run.ct@*"), + L1 = filelib:wildcard(WC), + 6 = length(L1), + + %% Keep all logs + {1,0,{0,0}}=ct_test_support:run_ct_run_test([{keep_logs,all}|Opts1], Config), + L2 = filelib:wildcard(WC), + 7 = length(L2), + 0 = ct_test_support:run_ct_script_start([{keep_logs,all}|Opts1], Config), + L3 = filelib:wildcard(WC), + 8 = length(L3), + + %% N<length of list + {1,0,{0,0}}=ct_test_support:run_ct_run_test([{keep_logs,7}|Opts1], Config), + L4 = filelib:wildcard(WC), + 7 = length(L4), + 0 = ct_test_support:run_ct_script_start([{keep_logs,6}|Opts1], Config), + L5 = filelib:wildcard(WC), + 6 = length(L5), + + %% N>length of list + {1,0,{0,0}}=ct_test_support:run_ct_run_test([{keep_logs,10}|Opts1], Config), + L6 = filelib:wildcard(WC), + 7 = length(L6), + 0 = ct_test_support:run_ct_script_start([{keep_logs,10}|Opts1], Config), + L7 = filelib:wildcard(WC), + 8 = length(L7), + + %% N==length of list + {1,0,{0,0}}=ct_test_support:run_ct_run_test([{keep_logs,8}|Opts1], Config), + L8 = filelib:wildcard(WC), + 8 = length(L8), + 0 = ct_test_support:run_ct_script_start([{keep_logs,8}|Opts1], Config), + L9 = filelib:wildcard(WC), + 8 = length(L9), + + %% N==length of list + current run + {1,0,{0,0}}=ct_test_support:run_ct_run_test([{keep_logs,9}|Opts1], Config), + L10 = filelib:wildcard(WC), + 9 = length(L10), + 0 = ct_test_support:run_ct_script_start([{keep_logs,10}|Opts1], Config), + L11 = filelib:wildcard(WC), + 10 = length(L11), + + {ok,Content} = file:list_dir(KeepLogsDir), + ct:log("Deleting dir: ~p~nContent: ~p~n",[KeepLogsDir,Content]), + ct_test_support:rm_dir(KeepLogsDir). + +%% Test the keep_logs option togwther with the refresh_logs option +refresh_logs(Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, "keep_logs_SUITE"), + Opts0 = ct_test_support:get_opts(Config), + LogDir=?config(logdir,Opts0), + KeepLogsDir = create_dir(filename:join(LogDir,"refresh_logs-")), + Opts1 = lists:keyreplace(logdir,1,Opts0,{logdir,KeepLogsDir}), + ct:log("New LogDir = ~s", [KeepLogsDir]), + + %% Create 6 ct_run.* log directories + SuiteOpts = [{suite,Suite},{label,refresh_logs} | Opts1], + [ok = ct_test_support:run(SuiteOpts, Config) || _ <- lists:seq(1,3)], + + %% Verify the number of directories + WC = filename:join(KeepLogsDir,"ct_run.ct@*"), + L1 = filelib:wildcard(WC), + 6 = length(L1), + + RefreshOpts = [{refresh_logs,KeepLogsDir},{label,refresh_logs} | Opts1], + + %% Keep all logs (note that refresh_logs option prevents the + %% creation of a new log directory for the current run) + done = ct_test_support:run_ct_run_test([{keep_logs,all}|RefreshOpts], Config), + L2 = filelib:wildcard(WC), + 6 = length(L2), + 0 = ct_test_support:run_ct_script_start([{keep_logs,all}|RefreshOpts],Config), + L3 = filelib:wildcard(WC), + 6 = length(L3), + + %% N<length of list + done = ct_test_support:run_ct_run_test([{keep_logs,5}|RefreshOpts], Config), + L5 = filelib:wildcard(WC), + 5 = length(L5), + 0 = ct_test_support:run_ct_script_start([{keep_logs,4}|RefreshOpts], Config), + L6 = filelib:wildcard(WC), + 4 = length(L6), + + %% N>length of list + done = ct_test_support:run_ct_run_test([{keep_logs,5}|RefreshOpts], Config), + L7 = filelib:wildcard(WC), + 4 = length(L7), + 0 = ct_test_support:run_ct_script_start([{keep_logs,5}|RefreshOpts], Config), + L8 = filelib:wildcard(WC), + 4 = length(L8), + + %% N==length of list + done = ct_test_support:run_ct_run_test([{keep_logs,4}|RefreshOpts], Config), + L9 = filelib:wildcard(WC), + 4 = length(L9), + 0 = ct_test_support:run_ct_script_start([{keep_logs,4}|RefreshOpts], Config), + L10 = filelib:wildcard(WC), + 4 = length(L10), + + {ok,Content} = file:list_dir(KeepLogsDir), + ct:log("Deleting dir: ~p~nContent: ~p~n",[KeepLogsDir,Content]), + ct_test_support:rm_dir(KeepLogsDir). + +%%%----------------------------------------------------------------- +%%% Internal +create_dir(Prefix) -> + I = erlang:unique_integer([positive]), + Dir = Prefix ++ integer_to_list(I), + case filelib:is_dir(Dir) of + true -> + %% Try again + create_dir(Prefix); + false -> + ok = file:make_dir(Dir), + Dir + end. diff --git a/lib/common_test/test/ct_keep_logs_SUITE_data/keep_logs_SUITE.erl b/lib/common_test/test/ct_keep_logs_SUITE_data/keep_logs_SUITE.erl new file mode 100644 index 0000000000..c78080748b --- /dev/null +++ b/lib/common_test/test/ct_keep_logs_SUITE_data/keep_logs_SUITE.erl @@ -0,0 +1,32 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(keep_logs_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> + []. +all() -> + [test_case]. + +test_case(_Config) -> + ok. diff --git a/lib/common_test/test/ct_log_SUITE.erl b/lib/common_test/test/ct_log_SUITE.erl index 9bdd44cbdf..93affda398 100644 --- a/lib/common_test/test/ct_log_SUITE.erl +++ b/lib/common_test/test/ct_log_SUITE.erl @@ -86,18 +86,7 @@ print(Config) -> io:format("5. Printing a pid: ~w~n", [Pid]), io:format("6. Printing HTML: <pre>~s</pre>~n", [String]), - %% --- API --- - %% pal(Format) -> - %% = ct:pal(default, 50, Format, []). - %% pal(X1, X2) -> ok - %% X1 = Category | Importance | Format - %% X2 = Format | FormatArgs - %% pal(X1, X2, X3) -> ok - %% X1 = Category | Importance - %% X2 = Importance | Format - %% X3 = Format | FormatArgs - %% pal(Category, Importance, Format, FormatArgs) -> ok - %% ------ + %% ct:pal ct:pal("1. Printing nothing"), ct:pal("2. Printing nothing", []), ct:pal("3. Printing a string: ~s", [String]), @@ -111,23 +100,16 @@ print(Config) -> ct:pal(50, "11. Printing with ~s", ["importance"]), ct:pal(ct_internal, 50, "12. Printing with ~s", ["category and importance"]), - %% --- API --- - %% log(Format) -> ok - %% = ct:log(default, 50, Format, [], []). - %% log(X1, X2) -> ok - %% X1 = Category | Importance | Format - %% X2 = Format | FormatArgs - %% log(X1, X2, X3) -> ok - %% X1 = Category | Importance - %% X2 = Importance | Format - %% X3 = Format | FormatArgs | Opts - %% log(X1, X2, X3, X4) -> ok - %% X1 = Category | Importance - %% X2 = Importance | Format - %% X3 = Format | FormatArgs - %% X4 = FormatArgs | Opts - %% log(Category, Importance, Format, FormatArgs, Opts) -> ok - %% ------ + ct:pal("13. Printing with heading", [], + [{heading,"This is a heading"}]), + ct:pal(ct_internal, "14. Printing with category and heading", [], + [{heading,"This is a heading"}]), + ct:pal(50, "15. Printing with importance and heading", [], + [{heading,"This is a heading"}]), + ct:pal(ct_internal, 50, "16. Printing with category, importance and heading", [], + [{heading,"This is a heading"}]), + + %% ct:log ct:log("1. Printing nothing"), ct:log("2. Printing nothing", []), ct:log("3. Printing a string: ~s", [String]), @@ -153,8 +135,37 @@ print(Config) -> ct:log(ct_internal, 50, "21. Printing a pid escaped with ~s, no_css: ~w", ["category and importance",Pid], [esc_chars,no_css]), + ct:log("22. Printing with heading", [], + [{heading,"This is a heading"}]), + ct:log(ct_internal, "23. Printing with category and heading", [], + [{heading,"This is a heading"}]), + ct:log(50, "24. Printing with importance and heading", [], + [{heading,"This is a heading"}]), + ct:log(ct_internal, 50, "25. Printing with category, importance and heading", [], + [{heading,"This is a heading"}]), + %% END mark ct:log("LOGGING END", [], [no_css]), + + + %% ct:print + ct:print("1. Does this show??"), + ct:print("2. Does this ~s", ["show??"]), + ct:print("3. Is this a non-html pid?? ~w", [self()]), + ct:print(ct_internal, "4. Printing with category"), + ct:print(ct_internal, "5. Printing with ~s", ["category"]), + ct:print(50, "6. Printing with importance"), + ct:print(50, "7. Printing with ~s", ["importance"]), + ct:print(ct_internal, 50, "8. Printing with ~s", ["category and importance"]), + ct:print("9. Printing with heading", [], + [{heading,"This is a heading"}]), + ct:print(ct_internal, "10. Printing with category and heading", [], + [{heading,"This is a heading"}]), + ct:print(50, "11. Printing with importance and heading", [], + [{heading,"This is a heading"}]), + ct:print(ct_internal, 50, "12. Printing with category, importance and heading", [], + [{heading,"This is a heading"}]), + {save_config,[{the_logfile,TcLogFile},{the_pid,Pid},{the_string,String}]}. @@ -169,6 +180,8 @@ verify(Config) -> {ok,Dev} = file:open(TcLogFile, [read]), ok = read_until(Dev, "LOGGING START\n"), + ct:pal("VERIFYING LOG ENTRIES...", []), + %% io:format match_line(Dev, "1. Printing nothing", []), read_nl(Dev), @@ -182,6 +195,7 @@ verify(Config) -> read_nl(Dev), match_line(Dev, "6. Printing HTML: <pre>~s</pre>", [String]), read_nl(Dev), + %% ct:pal read_header(Dev), match_line(Dev, "1. Printing nothing", []), @@ -219,6 +233,19 @@ verify(Config) -> read_header(Dev, "\"ct_internal\""), match_line(Dev, "12. Printing with ~s", ["category and importance"]), read_footer(Dev), + read_header(Dev, "\"default\"", "This is a heading"), + match_line(Dev, "13. Printing with heading", []), + read_footer(Dev), + read_header(Dev, "\"ct_internal\"", "This is a heading"), + match_line(Dev, "14. Printing with category and heading", []), + read_footer(Dev), + read_header(Dev, "\"default\"", "This is a heading"), + match_line(Dev, "15. Printing with importance and heading", []), + read_footer(Dev), + read_header(Dev, "\"ct_internal\"", "This is a heading"), + match_line(Dev, "16. Printing with category, importance and heading", []), + read_footer(Dev), + %% ct:log read_header(Dev), match_line(Dev, "1. Printing nothing", []), @@ -275,7 +302,18 @@ verify(Config) -> read_footer(Dev), match_line(Dev, "21. Printing a pid escaped with ~s, no_css: ~s", ["category and importance",EscPid]), - + read_header(Dev, "\"default\"", "This is a heading"), + match_line(Dev, "22. Printing with heading", []), + read_footer(Dev), + read_header(Dev, "\"ct_internal\"", "This is a heading"), + match_line(Dev, "23. Printing with category and heading", []), + read_footer(Dev), + read_header(Dev, "\"default\"", "This is a heading"), + match_line(Dev, "24. Printing with importance and heading", []), + read_footer(Dev), + read_header(Dev, "\"ct_internal\"", "This is a heading"), + match_line(Dev, "25. Printing with category, importance and heading", []), + read_footer(Dev), file:close(Dev), ok. @@ -298,29 +336,51 @@ read_until(Dev, Pat) -> match_line(Dev, Format, Args) -> Pat = lists:flatten(io_lib:format(Format, Args)), Line = element(2, file:read_line(Dev)), + + %% for debugging purposes: + ct:pal("L: ~tp", [Line], [no_css]), + case re:run(Line, Pat) of {match,_} -> ok; nomatch -> - ct:pal("ERROR! No match for ~p.\nLine = ~p", [Pat,Line]), + ct:pal("ERROR! No match for ~p", [Pat]), file:close(Dev), ct:fail({mismatch,Pat,Line}) end. read_header(Dev) -> - read_header(Dev, "\"default\""). + read_header(Dev, "\"default\"", "User"). read_header(Dev, Cat) -> + read_header(Dev, Cat, "User"). + +read_header(Dev, Cat, Heading) -> file:read_line(Dev), % \n "</pre>\n" = element(2, file:read_line(Dev)), - {match,_} = - re:run(element(2, file:read_line(Dev)), "<div class="++Cat++"><pre><b>" - "\\*\\*\\* User \\d{4}-\\d{2}-\\d{2} " - "\\d{2}:\\d{2}:\\d{2}.\\d{1,} \\*\\*\\*</b>"). + {ok,Hd} = file:read_line(Dev), + + %% for debugging purposes: + ct:pal("H: ~tp", [Hd], [no_css]), + + Pat = "<div class="++Cat++"><pre><b>"++ + "\\*\\*\\* "++Heading++" \\d{4}-\\d{2}-\\d{2} "++ + "\\d{2}:\\d{2}:\\d{2}.\\d{1,} \\*\\*\\*</b>", + + case re:run(Hd, Pat) of + {match,_} -> + ok; + _ -> + ct:pal("ERROR! No match for ~p", [Pat]), + file:close(Dev), + ct:fail({mismatch,Pat,Hd}) + end. read_footer(Dev) -> "</pre></div>\n" = element(2, file:read_line(Dev)), - "<pre>\n" = element(2, file:read_line(Dev)). + "<pre>\n" = element(2, file:read_line(Dev)), + %% for debugging purposes: + ct:pal("F: </pre></div><pre>", [], [no_css]). read_nl(Dev) -> file:read_line(Dev). diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl index 8932f930d1..05edb45fe8 100644 --- a/lib/common_test/test/ct_netconfc_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE.erl @@ -52,9 +52,8 @@ init_per_suite(Config) -> end. check_crypto_and_ssh() -> - (catch code:load_file(crypto)), - case code:is_loaded(crypto) of - {file,_} -> + case code:ensure_loaded(crypto) of + {module,_} -> case catch ssh:start() of Ok when Ok==ok; Ok=={error,{already_started,ssh}} -> ct:log("ssh started",[]), diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl index 2aa6c4d354..6a41f0a04c 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl @@ -102,7 +102,9 @@ all() -> receive_one_event, receive_multiple_events, receive_event_and_rpc, - receive_event_and_rpc_in_chunks + receive_event_and_rpc_in_chunks, + multiple_channels, + kill_session_same_connection ] end. @@ -124,7 +126,7 @@ end_per_testcase(_Case, _Config) -> ok. init_per_suite(Config) -> - (catch code:load_file(crypto)), + code:ensure_loaded(crypto), case {ssh:start(),code:is_loaded(crypto)} of {Ok,{file,_}} when Ok==ok; Ok=={error,{already_started,ssh}} -> ct:log("ssh started",[]), @@ -498,10 +500,11 @@ kill_session(Config) -> ?NS:hello(2), ?NS:expect(2,hello), - {ok,_OtherClient} = open(SshDir), + {ok,OtherClient} = open(SshDir), ?NS:expect_do_reply('kill-session',{kill,2},ok), ?ok = ct_netconfc:kill_session(Client,2), + {error,_}=ct_netconfc:get(OtherClient,{server,[{xmlns,"myns"}],[]}), ?NS:expect_do_reply('close-session',close,ok), ?ok = ct_netconfc:close_session(Client), @@ -1179,13 +1182,73 @@ receive_event_and_rpc_in_chunks(Config) -> ?ok = ct_netconfc:close_session(Client), ok. +multiple_channels(Config) -> + SshDir = ?config(ssh_dir,Config), + SshOpts = ?DEFAULT_SSH_OPTS(SshDir), + {ok,Conn} = ct_netconfc:connect(SshOpts), + ?NS:hello(1), + ?NS:expect(hello), + {ok,Client1} = ct_netconfc:session(Conn), + ?NS:hello(2), + ?NS:expect(2,hello), + {ok,Client2} = ct_netconfc:session(Conn), + ?NS:hello(3), + ?NS:expect(3,hello), + {ok,Client3} = ct_netconfc:session(Conn), + + Data = [{server,[{xmlns,"myns"}],[{name,[],["myserver"]}]}], + ?NS:expect_reply(1,'get',{data,Data}), + {ok,Data} = ct_netconfc:get(Client1,{server,[{xmlns,"myns"}],[]}), + ?NS:expect_reply(2,'get',{data,Data}), + {ok,Data} = ct_netconfc:get(Client2,{server,[{xmlns,"myns"}],[]}), + ?NS:expect_reply(3,'get',{data,Data}), + {ok,Data} = ct_netconfc:get(Client3,{server,[{xmlns,"myns"}],[]}), + + ?NS:expect_do_reply(2,'close-session',close,ok), + ?ok = ct_netconfc:close_session(Client2), + + ?NS:expect_reply(1,'get',{data,Data}), + {ok,Data} = ct_netconfc:get(Client1,{server,[{xmlns,"myns"}],[]}), + {error,no_such_client}=ct_netconfc:get(Client2,{server,[{xmlns,"myns"}],[]}), + ?NS:expect_reply(3,'get',{data,Data}), + {ok,Data} = ct_netconfc:get(Client3,{server,[{xmlns,"myns"}],[]}), + + ?NS:expect_do_reply(1,'close-session',close,ok), + ?ok = ct_netconfc:close_session(Client1), + ?NS:expect_do_reply(3,'close-session',close,ok), + ?ok = ct_netconfc:close_session(Client3), + + ?ok = ct_netconfc:disconnect(Conn), + ok. + +kill_session_same_connection(Config) -> + SshDir = ?config(ssh_dir,Config), + SshOpts = ?DEFAULT_SSH_OPTS(SshDir), + {ok,Conn} = ct_netconfc:connect(SshOpts), + ?NS:hello(1), + ?NS:expect(hello), + {ok,Client1} = ct_netconfc:session(Conn), + ?NS:hello(2), + ?NS:expect(2,hello), + {ok,Client2} = ct_netconfc:session(Conn), + + ?NS:expect_do_reply('kill-session',{kill,2},ok), + ?ok = ct_netconfc:kill_session(Client1,2), + timer:sleep(1000), + {error,no_such_client}=ct_netconfc:get(Client2,{server,[{xmlns,"myns"}],[]}), + + ?NS:expect_do_reply('close-session',close,ok), + ?ok = ct_netconfc:close_session(Client1), + + ok. + %%%----------------------------------------------------------------- break(_Config) -> - test_server:break("break test case"). + ct:break("break test case"). br() -> - test_server:break(""). + ct:break(""). %%%----------------------------------------------------------------- %% Open a netconf session which is not specified in a config file diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc_remote_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc_remote_SUITE.erl index f2580ad8e9..3ce2d18c66 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc_remote_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc_remote_SUITE.erl @@ -62,7 +62,7 @@ stop_node(Case) -> init_per_suite(Config) -> - (catch code:load_file(crypto)), + code:ensure_loaded(crypto), case {ssh:start(),code:is_loaded(crypto)} of {Ok,{file,_}} when Ok==ok; Ok=={error,{already_started,ssh}} -> ct:log("SSH started locally",[]), diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl index 2412ea6aba..c40bf9e2cc 100644 --- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl +++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl @@ -254,7 +254,7 @@ data(Data, State = #session{connection = ConnRef, end. stop_channel(CM, Ch, State) -> - ssh:close(CM), + ssh_connection:close(CM,Ch), {stop, Ch, State}. @@ -290,8 +290,8 @@ send_frag({CM,Ch},Data) -> %%% Kill ssh connection -kill({CM,_Ch}) -> - ssh:close(CM). +kill({CM,Ch}) -> + ssh_connection:close(CM,Ch). add_expect(SessionId,Add) -> table_trans(fun do_add_expect/2,[SessionId,Add]). diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index f8b6a379f6..9382e4c011 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2013-2016. All Rights Reserved. +%% Copyright Ericsson AB 2013-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -363,14 +363,17 @@ skip_first_tc1(Suite) -> {?eh,tc_start,{Suite,tc1}}, {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,1}}}, + {?eh,tc_start,{Suite,{init_per_group,g,[]}}}, {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}}, {?eh,tc_auto_skip,{Suite,{tc1,g},?skip_reason}}, {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_auto_skip,{Suite,{tc2,g},?skip_reason}}, {?eh,test_stats,{'_',0,{0,3}}}, {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}}, + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,4}}}, {?eh,tc_start,{Suite,end_per_suite}}, @@ -390,10 +393,12 @@ skip_tc1_in_group(Suite) -> {?eh,tc_start,{Suite,tc1}}, {?eh,tc_done,{Suite,tc1,ok}}, {?eh,test_stats,{'_',0,{0,0}}}, + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,1}}}, {?eh,tc_start,{Suite,{end_per_group,g,[]}}}, {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}], + {?eh,tc_start,{Suite,tc2}}, {?eh,tc_done,{Suite,tc2,?skipped}}, {?eh,test_stats,{'_',0,{0,2}}}, {?eh,tc_start,{Suite,end_per_suite}}, diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index 42ec685c16..832e105517 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012-2016. All Rights Reserved. +%% Copyright Ericsson AB 2012-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -73,7 +73,9 @@ all() -> relative_path, url, logdir, - fail_pre_init_per_suite + fail_pre_init_per_suite, + skip_case_in_spec, + skip_suite_in_spec ]. %%-------------------------------------------------------------------- @@ -119,6 +121,18 @@ fail_pre_init_per_suite(Config) when is_list(Config) -> run(fail_pre_init_per_suite,[fail_pre_init_per_suite, {cth_surefire,[{path,Path}]}],Path,Config,[],Suites). +skip_case_in_spec(Config) -> + DataDir = ?config(data_dir,Config), + Spec = filename:join(DataDir,"skip_one_case.spec"), + Path = "skip_case_in_spec.xml", + run_spec(skip_case_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec). + +skip_suite_in_spec(Config) -> + DataDir = ?config(data_dir,Config), + Spec = filename:join(DataDir,"skip_one_suite.spec"), + Path = "skip_suite_in_spec.xml", + run_spec(skip_suite_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -129,8 +143,15 @@ run(Case,CTHs,Report,Config,ExtraOpts) -> Suite = filename:join(DataDir, "surefire_SUITE"), run(Case,CTHs,Report,Config,ExtraOpts,Suite). run(Case,CTHs,Report,Config,ExtraOpts,Suite) -> - {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], - Config), + Test = [{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], + do_run(Case, Report, Test, Config). + +run_spec(Case,CTHs,Report,Config,Spec) -> + Test = [{spec,Spec},{ct_hooks,CTHs},{label,Case}], + do_run(Case, Report, Test, Config). + +do_run(Case, Report, Test, Config) -> + {Opts,ERPid} = setup(Test, Config), ok = execute(Case, Opts, ERPid, Config), LogDir = case lists:keyfind(logdir,1,Opts) of @@ -201,7 +222,10 @@ test_suite_events(pass_SUITE) -> {?eh,test_stats,{1,0,{0,0}}}, {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,tc_done,{ct_framework,end_per_suite,ok}}]; -test_suite_events(_) -> +test_suite_events(skip_all_surefire_SUITE) -> + [{?eh,tc_user_skip,{skip_all_surefire_SUITE,all,"skipped in spec"}}, + {?eh,test_stats,{0,0,{1,0}}}]; +test_suite_events(Test) -> [{?eh,tc_start,{surefire_SUITE,init_per_suite}}, {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}}, {?eh,tc_start,{surefire_SUITE,tc_ok}}, @@ -210,46 +234,55 @@ test_suite_events(_) -> {?eh,tc_start,{surefire_SUITE,tc_fail}}, {?eh,tc_done,{surefire_SUITE,tc_fail, {failed,{error,{test_case_failed,"this test should fail"}}}}}, - {?eh,test_stats,{1,1,{0,0}}}, - {?eh,tc_start,{surefire_SUITE,tc_skip}}, - {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, - {?eh,test_stats,{1,1,{1,0}}}, - {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, - {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {auto_skipped,{require_failed,'_'}}}}, - {?eh,test_stats,{1,1,{1,1}}}, - [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, - {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, - {?eh,tc_start,{surefire_SUITE,tc_ok}}, - {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, - {?eh,test_stats,{2,1,{1,1}}}, - {?eh,tc_start,{surefire_SUITE,tc_fail}}, - {?eh,tc_done,{surefire_SUITE,tc_fail, - {failed,{error,{test_case_failed,"this test should fail"}}}}}, - {?eh,test_stats,{2,2,{1,1}}}, - {?eh,tc_start,{surefire_SUITE,tc_skip}}, - {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, - {?eh,test_stats,{2,2,{2,1}}}, - {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, - {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, - {auto_skipped,{require_failed,'_'}}}}, - {?eh,test_stats,{2,2,{2,2}}}, - {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, - {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], - [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, - {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, - {failed,{error,all_cases_should_be_skipped}}}}, - {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail}, - {failed, - {surefire_SUITE,init_per_group, - {'EXIT',all_cases_should_be_skipped}}}}}, - {?eh,test_stats,{2,2,{2,3}}}, - {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, - {failed, - {surefire_SUITE,init_per_group, - {'EXIT',all_cases_should_be_skipped}}}}}], - {?eh,tc_start,{surefire_SUITE,end_per_suite}}, - {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}]. + {?eh,test_stats,{1,1,{0,0}}}] ++ + tc_skip_events(Test,undefined) ++ + [{?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {auto_skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{1,1,{1,1}}}, + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, + {?eh,tc_start,{surefire_SUITE,tc_ok}}, + {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, + {?eh,test_stats,{2,1,{1,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_fail}}, + {?eh,tc_done,{surefire_SUITE,tc_fail, + {failed,{error,{test_case_failed,"this test should fail"}}}}}, + {?eh,test_stats,{2,2,{1,1}}}] ++ + tc_skip_events(Test,g) ++ + [{?eh,test_stats,{2,2,{2,1}}}, + {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, + {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, + {auto_skipped,{require_failed,'_'}}}}, + {?eh,test_stats,{2,2,{2,2}}}, + {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, + {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], + [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, + {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, + {failed,{error,all_cases_should_be_skipped}}}}, + {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail}, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}, + {?eh,test_stats,{2,2,{2,3}}}, + {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, + {failed, + {surefire_SUITE,init_per_group, + {'EXIT',all_cases_should_be_skipped}}}}}], + {?eh,tc_start,{surefire_SUITE,end_per_suite}}, + {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}]. + +tc_skip_events(skip_case_in_spec,Group) -> + [{?eh,tc_user_skip,{surefire_SUITE,tc_skip_name(Group),"skipped in spec"}}]; +tc_skip_events(_Test,_Group) -> + [{?eh,tc_start,{surefire_SUITE,tc_skip}}, + {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}]. + +tc_skip_name(undefined) -> + tc_skip; +tc_skip_name(Group) -> + {tc_skip,Group}. test_events(fail_pre_init_per_suite) -> [{?eh,start_logging,{'DEF','RUNDIR'}}, @@ -257,6 +290,10 @@ test_events(fail_pre_init_per_suite) -> test_suite_events(pass_SUITE) ++ test_suite_events(fail_SUITE, {1,0,{0,1}}) ++ [{?eh,stop_logging,[]}]; +test_events(skip_suite_in_spec) -> + [{?eh,start_logging,'_'},{?eh,start_info,{1,1,0}}] ++ + test_suite_events(skip_all_surefire_SUITE) ++ + [{?eh,stop_logging,[]}]; test_events(Test) -> [{?eh,start_logging,'_'}, {?eh,start_info,{1,1,9}}] ++ test_suite_events(Test) ++ @@ -364,6 +401,8 @@ failed_or_skipped([]) -> events_to_result(E) -> events_to_result(E, []). +events_to_result([{?eh,tc_user_skip,{_Suite,all,_}}|E], Result) -> + events_to_result(E, [[[s]]|Result]); events_to_result([{?eh,tc_auto_skip,{_Suite,init_per_suite,_}}|E], Result) -> {Suite,Rest} = events_to_result1(E), events_to_result(Rest, [[[s]|Suite]|Result]); @@ -382,7 +421,7 @@ events_to_result1([{?eh,tc_done,{_Suite, end_per_suite,R}}|E]) -> events_to_result1([{?eh,tc_done,{_Suite,_Case,R}}|E]) -> {Suite,Rest} = events_to_result1(E), {[result(R)|Suite],Rest}; -events_to_result1([{?eh,tc_auto_skip,_}|E]) -> +events_to_result1([{?eh,Skip,_}|E]) when Skip==tc_auto_skip; Skip==tc_user_skip -> {Suite,Rest} = events_to_result1(E), {[[s]|Suite],Rest}; events_to_result1([_|E]) -> diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec new file mode 100644 index 0000000000..42df8a7d1a --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec @@ -0,0 +1,2 @@ +{suites,".",surefire_SUITE}. +{skip_cases,".",surefire_SUITE,tc_skip,"skipped in spec"}. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec new file mode 100644 index 0000000000..57966328ab --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec @@ -0,0 +1,2 @@ +{suites,".",[skip_all_surefire_SUITE]}. +{skip_suites,".",skip_all_surefire_SUITE,"skipped in spec"}. diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 228d900545..0455313669 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2016. All Rights Reserved. +%% Copyright Ericsson AB 2009-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -161,6 +161,7 @@ test_events(ts_if_1) -> {?eh,tc_start,{ts_if_1_SUITE,tc4}}, {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}}, {?eh,test_stats,{1,2,{0,1}}}, + {?eh,tc_start,{ts_if_1_SUITE,tc5}}, {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}}, {?eh,test_stats,{1,2,{0,2}}}, diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index e926abd885..ba7aadfeec 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2016. All Rights Reserved. +%% Copyright Ericsson AB 2008-2017. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -45,6 +45,8 @@ -export([unique_timestamp/0]). +-export([rm_dir/1]). + -include_lib("kernel/include/file.hrl"). %%%----------------------------------------------------------------- @@ -351,6 +353,9 @@ check_result(CtRunTestResult,ExitStatus,Opts) catch _:_ -> {error,{unexpected_return_value,{CtRunTestResult,ExitStatus}}} end; +check_result(done,0,_Opts) -> + %% refresh_logs return + ok; check_result(CtRunTestResult,ExitStatus,_Opts) -> {error,{unexpected_return_value,{CtRunTestResult,ExitStatus}}}. @@ -765,23 +770,23 @@ locate({parallel,TEvs}, Node, Evs, Config) -> {Done,RemEvs2,length(RemEvs2)} end; %% end_per_group auto- or user skipped - (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize}) when AutoOrUserSkip == tc_auto_skip; AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G -> case match_data(R, Reason) of match -> false; _ -> true end; ({EH,#event{name=tc_user_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G -> case match_data(R, Reason) of match -> false; _ -> true @@ -1008,20 +1013,20 @@ locate({shuffle,TEvs}, Node, Evs, Config) -> {Done,RemEvs2,length(RemEvs2)} end; %% end_per_group auto-or user skipped - (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) + (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize}) when AutoOrUserSkip == tc_auto_skip; AutoOrUserSkip == tc_user_skip -> RemEvs1 = lists:dropwhile( fun({EH,#event{name=tc_auto_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R -> false; ({EH,#event{name=tc_user_skip, node=EvNode, - data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> + data={Mod,{end_per_group,EvGroupName},Reason}}}) when + EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R -> false; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when @@ -1264,10 +1269,10 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev, io:format(Dev, "~s~p]},~n", [Ind,E]), log_events1(Evs, Dev, Ind--" ") end; -log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> +log_events1([E={_EH,tc_auto_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); -log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> +log_events1([E={_EH,tc_user_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) -> io:format(Dev, "~s~p],~n", [Ind,E]), log_events1(Evs, Dev, Ind--" "); log_events1([E], Dev, Ind) -> diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl index 1a941df185..1bab80942a 100644 --- a/lib/common_test/test/ct_testspec_2_SUITE.erl +++ b/lib/common_test/test/ct_testspec_2_SUITE.erl @@ -220,7 +220,24 @@ basic_compatible_no_nodes(_Config) -> {tc2,{skip,"skipped"}}]}]}], merge_tests = true}, - verify_result(Verify,ListResult,FileResult). + verify_result(Verify,ListResult,FileResult), + + {ok,Tests} = ct_testspec:get_tests([SpecFile]), + ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]), + [{[SpecFile],[{Node,Run,Skip}]}] = Tests, + [{Alias1V,x_SUITE,all}, + {Alias1V,y_SUITE,[{g1,all},{g2,all},tc1,tc2]}, + {Alias1V,z_SUITE,all}, + {Alias2V,x_SUITE,all}, + {Alias2V,y_SUITE,all}] = lists:sort(Run), + [{Alias1V,z_SUITE,"skipped"}, + {Alias2V,x_SUITE,{g1,all},"skipped"}, + {Alias2V,x_SUITE,{g2,all},"skipped"}, + {Alias2V,y_SUITE,tc1,"skipped"}, + {Alias2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip), + + ok. + %%%----------------------------------------------------------------- %%% @@ -346,7 +363,25 @@ basic_compatible_nodes(_Config) -> {tc2,{skip,"skipped"}}]}]}], merge_tests = true}, - verify_result(Verify,ListResult,FileResult). + verify_result(Verify,ListResult,FileResult), + + {ok,Tests} = ct_testspec:get_tests([SpecFile]), + ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]), + [{[SpecFile],[{Node,[],[]}, + {Node1,Run1,Skip1}, + {Node2,Run2,Skip2}]}] = Tests, + [{TO1V,x_SUITE,all}, + {TO1V,y_SUITE,[{g1,all},{g2,all},tc1,tc2]}, + {TO1V,z_SUITE,all}] = lists:sort(Run1), + [{TO2V,x_SUITE,all}, + {TO2V,y_SUITE,all}] = lists:sort(Run2), + [{TO1V,z_SUITE,"skipped"}] = lists:sort(Skip1), + [{TO2V,x_SUITE,{g1,all},"skipped"}, + {TO2V,x_SUITE,{g2,all},"skipped"}, + {TO2V,y_SUITE,tc1,"skipped"}, + {TO2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip2), + + ok. %%%----------------------------------------------------------------- %%% @@ -439,7 +474,28 @@ no_merging(_Config) -> [{y_SUITE,[{tc1,{skip,"skipped"}}, {tc2,{skip,"skipped"}}]}]}]}, - verify_result(Verify,ListResult,FileResult). + verify_result(Verify,ListResult,FileResult), + + {ok,Tests} = ct_testspec:get_tests([SpecFile]), + ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]), + [{[SpecFile],[{Node,[],[]}, + {Node1,Run1,Skip1}, + {Node2,Run2,Skip2}]}] = Tests, + [{TO1V,x_SUITE,all}, + {TO1V,y_SUITE,[tc1,tc2]}, + {TO1V,y_SUITE,[{g1,all},{g2,all}]}, + {TO1V,z_SUITE,all}] = lists:sort(Run1), + [{TO2V,x_SUITE,all}, + {TO2V,x_SUITE,[{skipped,g1,all},{skipped,g2,all}]}, + {TO2V,y_SUITE,all}, + {TO2V,y_SUITE,[{skipped,tc1},{skipped,tc2}]}] = lists:sort(Run2), + [{TO1V,z_SUITE,"skipped"}] = lists:sort(Skip1), + [{TO2V,x_SUITE,{g1,all},"skipped"}, + {TO2V,x_SUITE,{g2,all},"skipped"}, + {TO2V,y_SUITE,tc1,"skipped"}, + {TO2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip2), + + ok. %%%----------------------------------------------------------------- %%% @@ -510,7 +566,25 @@ multiple_specs(_Config) -> {y_SUITE,[all,{tc1,{skip,"skipped"}}, {tc2,{skip,"skipped"}}]}]}]}, - verify_result(Verify,FileResult,FileResult). + verify_result(Verify,FileResult,FileResult), + + {ok,Tests} = ct_testspec:get_tests([[SpecFile1,SpecFile2]]), + ct:pal("ct_testspec:get_tests/1:~n~p~n", [Tests]), + [{[SpecFile1,SpecFile2],[{Node,[],[]}, + {Node1,Run1,Skip1}, + {Node2,Run2,Skip2}]}] = Tests, + [{TO1V,x_SUITE,all}, + {TO1V,y_SUITE,[{g1,all},{g2,all},tc1,tc2]}, + {TO1V,z_SUITE,all}] = lists:sort(Run1), + [{TO2V,x_SUITE,all}, + {TO2V,y_SUITE,all}] = lists:sort(Run2), + [{TO1V,z_SUITE,"skipped"}] = lists:sort(Skip1), + [{TO2V,x_SUITE,{g1,all},"skipped"}, + {TO2V,x_SUITE,{g2,all},"skipped"}, + {TO2V,y_SUITE,tc1,"skipped"}, + {TO2V,y_SUITE,tc2,"skipped"}] = lists:sort(Skip2), + + ok. %%%----------------------------------------------------------------- %%% diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk index ab5cfd7a80..a219aa4736 100644 --- a/lib/common_test/vsn.mk +++ b/lib/common_test/vsn.mk @@ -1 +1 @@ -COMMON_TEST_VSN = 1.12.3 +COMMON_TEST_VSN = 1.15 |