diff options
Diffstat (limited to 'lib/test_server')
37 files changed, 1203 insertions, 369 deletions
diff --git a/lib/test_server/doc/src/basics_chapter.xml b/lib/test_server/doc/src/basics_chapter.xml index 2e60d412e9..eba24615cb 100644 --- a/lib/test_server/doc/src/basics_chapter.xml +++ b/lib/test_server/doc/src/basics_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> diff --git a/lib/test_server/doc/src/book.xml b/lib/test_server/doc/src/book.xml index 960ce48cf7..006a2f167d 100644 --- a/lib/test_server/doc/src/book.xml +++ b/lib/test_server/doc/src/book.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE book SYSTEM "book.dtd"> <book xmlns:xi="http://www.w3.org/2001/XInclude"> <header titlestyle="normal"> <copyright> - <year>2002</year><year>2009</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/test_server/doc/src/example_chapter.xml b/lib/test_server/doc/src/example_chapter.xml index 8a06526528..0ebc85da09 100644 --- a/lib/test_server/doc/src/example_chapter.xml +++ b/lib/test_server/doc/src/example_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2002</year><year>2009</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/test_server/doc/src/fascicules.xml b/lib/test_server/doc/src/fascicules.xml index 0678195e07..37feca543f 100644 --- a/lib/test_server/doc/src/fascicules.xml +++ b/lib/test_server/doc/src/fascicules.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE fascicules SYSTEM "fascicules.dtd"> <fascicules> diff --git a/lib/test_server/doc/src/notes.xml b/lib/test_server/doc/src/notes.xml index c83fc9d82b..d05626094a 100644 --- a/lib/test_server/doc/src/notes.xml +++ b/lib/test_server/doc/src/notes.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> @@ -32,6 +32,54 @@ <file>notes.xml</file> </header> +<section><title>Test_Server 3.6.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>The way Common Test handles skipping of test cases has + been updated. In previous versions, returning + <c>{skip,Reason}</c> from a configuration function (such + as init_per_suite or init_per_group), resulted in all + affected test cases getting skipped with status + <c>auto_skipped</c>. This was inappropriate, since this + status is supposed to be used to inform that Common Test + has taken the initiative to skip something (e.g. a test + case group if init_per_group failed). Therefore, in this + version of Common Test, whenever the user skips a suite, + group, or individual test case (by means of a + configuration function or test specification term), the + affected test cases get the status <c>user_skipped</c> + instead.</p> <p>This update has meant a few changes that + may affect Common Test users in various ways: <list> + <item>The test results and statistics will be affected, + which is important to know when running regression tests + and comparing results to previous test runs.</item> + <item>Users that read or parse the textual log file + <c>suite.log</c> will notice that an auto skipped + function is now reported as <c>auto_skipped</c> rather + than <c>skipped</c> as before.</item> <item>When + <c>require</c> fails in an info function (such as suite/0 + or group/1), all affected configuration functions and + test cases are marked as <c>auto_skipped</c>.</item> + <item>If Common Test detects an error in the test suite + (such as e.g. an invalid all/0 function), all affected + configuration functions and test cases are marked as + <c>auto_skipped</c>.</item> <item>If a repeated test run + session reaches a deadline with <c>force_stop</c> + enabled, all remaining test cases are marked as + <c>auto_skipped</c> rather than <c>user_skipped</c> as + before.</item> <item>The event messages that Common Test + generates during test runs have been affected by this + update. For details see OTP-11524.</item> </list> </p> + <p> + Own Id: OTP-11305 Aux Id: OTP-11524 </p> + </item> + </list> + </section> + +</section> + <section><title>Test_Server 3.6.3</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/lib/test_server/doc/src/notes_history.xml b/lib/test_server/doc/src/notes_history.xml index 0392bd74a2..898717cbf0 100644 --- a/lib/test_server/doc/src/notes_history.xml +++ b/lib/test_server/doc/src/notes_history.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2006</year><year>2009</year> + <year>2006</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/test_server/doc/src/part.xml b/lib/test_server/doc/src/part.xml index a20835e035..43693ff9cb 100644 --- a/lib/test_server/doc/src/part.xml +++ b/lib/test_server/doc/src/part.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> <part xmlns:xi="http://www.w3.org/2001/XInclude"> diff --git a/lib/test_server/doc/src/part_notes.xml b/lib/test_server/doc/src/part_notes.xml index abf949dfda..e0656942cd 100644 --- a/lib/test_server/doc/src/part_notes.xml +++ b/lib/test_server/doc/src/part_notes.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> <part xmlns:xi="http://www.w3.org/2001/XInclude"> diff --git a/lib/test_server/doc/src/part_notes_history.xml b/lib/test_server/doc/src/part_notes_history.xml index 4746f583b9..d3ff49830f 100644 --- a/lib/test_server/doc/src/part_notes_history.xml +++ b/lib/test_server/doc/src/part_notes_history.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE part SYSTEM "part.dtd"> <part> diff --git a/lib/test_server/doc/src/ref_man.xml b/lib/test_server/doc/src/ref_man.xml index ca0b38e0c8..cc0e7eaf5c 100644 --- a/lib/test_server/doc/src/ref_man.xml +++ b/lib/test_server/doc/src/ref_man.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE application SYSTEM "application.dtd"> <application xmlns:xi="http://www.w3.org/2001/XInclude"> diff --git a/lib/test_server/doc/src/run_test_chapter.xml b/lib/test_server/doc/src/run_test_chapter.xml index 36bd41da1f..35344255d4 100644 --- a/lib/test_server/doc/src/run_test_chapter.xml +++ b/lib/test_server/doc/src/run_test_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2002</year><year>2009</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/test_server/doc/src/test_server.xml b/lib/test_server/doc/src/test_server.xml index 5ef43dd628..ed5569e1fe 100644 --- a/lib/test_server/doc/src/test_server.xml +++ b/lib/test_server/doc/src/test_server.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> @@ -625,6 +625,31 @@ Only valid for peer nodes. Note that slave nodes always </desc> </func> <func> + <name>appup_test(App) -> ok | test_server:fail()</name> + <fsummary>Checks an applications .appup file for obvious errors</fsummary> + <type> + <v>App = term()</v> + <d>The name of the application to test</d> + </type> + <desc> + <p>Checks an applications .appup file for obvious errors. + The following is checked: + </p> + <list type="bulleted"> + <item>syntax + </item> + <item>that .app file version and .appup file version match + </item> + <item>for non-library applications: validity of high-level upgrade + instructions, specifying no instructions is explicitly allowed + (in this case the application is not upgradeable)</item> + <item>for library applications: that there is exactly one wildcard + regexp clause restarting the application when upgrading or + downgrading from any version</item> + </list> + </desc> + </func> + <func> <name>comment(Comment) -> ok</name> <fsummary>Print a comment on the HTML result page</fsummary> <type> diff --git a/lib/test_server/doc/src/test_server_app.xml b/lib/test_server/doc/src/test_server_app.xml index ea4b5ffecf..e5481495a2 100644 --- a/lib/test_server/doc/src/test_server_app.xml +++ b/lib/test_server/doc/src/test_server_app.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE appref SYSTEM "appref.dtd"> <appref> diff --git a/lib/test_server/doc/src/test_server_ctrl.xml b/lib/test_server/doc/src/test_server_ctrl.xml index 9252cdce4f..f4aae724e0 100644 --- a/lib/test_server/doc/src/test_server_ctrl.xml +++ b/lib/test_server/doc/src/test_server_ctrl.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> diff --git a/lib/test_server/doc/src/test_spec_chapter.xml b/lib/test_server/doc/src/test_spec_chapter.xml index c197e03bed..342f772f0b 100644 --- a/lib/test_server/doc/src/test_spec_chapter.xml +++ b/lib/test_server/doc/src/test_spec_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> diff --git a/lib/test_server/doc/src/ts.xml b/lib/test_server/doc/src/ts.xml index 82ba3a5017..bd5a3cbaa3 100644 --- a/lib/test_server/doc/src/ts.xml +++ b/lib/test_server/doc/src/ts.xml @@ -1,11 +1,11 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE erlref SYSTEM "erlref.dtd"> <erlref> <header> <copyright> <year>2007</year> - <year>2012</year> + <year>2013</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/lib/test_server/doc/src/why_test_chapter.xml b/lib/test_server/doc/src/why_test_chapter.xml index 745d4218f1..9b0ffc6f12 100644 --- a/lib/test_server/doc/src/why_test_chapter.xml +++ b/lib/test_server/doc/src/why_test_chapter.xml @@ -1,10 +1,10 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> <header> <copyright> - <year>2002</year><year>2009</year> + <year>2002</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> diff --git a/lib/test_server/doc/src/write_framework_chapter.xml b/lib/test_server/doc/src/write_framework_chapter.xml index 5c1c657d9e..3a6fda9610 100644 --- a/lib/test_server/doc/src/write_framework_chapter.xml +++ b/lib/test_server/doc/src/write_framework_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> diff --git a/lib/test_server/doc/src/write_test_chapter.xml b/lib/test_server/doc/src/write_test_chapter.xml index 086ed2a18d..ce7f76549d 100644 --- a/lib/test_server/doc/src/write_test_chapter.xml +++ b/lib/test_server/doc/src/write_test_chapter.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> diff --git a/lib/test_server/src/configure.in b/lib/test_server/src/configure.in index 3815027721..cd723bcd4d 100644 --- a/lib/test_server/src/configure.in +++ b/lib/test_server/src/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script for Erlang. dnl dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1997-2013. All Rights Reserved. +dnl Copyright Ericsson AB 1997-2014. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -38,6 +38,35 @@ AC_ARG_ENABLE(debug-mode, *) CFLAGS=$DEBUG_FLAGS ;; esac ], ) +AC_ARG_ENABLE(m64-build, +AS_HELP_STRING([--enable-m64-build], + [build 64-bit binaries using the -m64 flag to (g)cc]), +[ case "$enableval" in + no) enable_m64_build=no ;; + *) enable_m64_build=yes ;; + esac +],enable_m64_build=no) + +AC_ARG_ENABLE(m32-build, +AS_HELP_STRING([--enable-m32-build], + [build 32-bit binaries using the -m32 flag to (g)cc]), +[ case "$enableval" in + no) enable_m32_build=no ;; + *) enable_m32_build=yes ;; + esac +],enable_m32_build=no) + +no_mXX_LDFLAGS="$LDFLAGS" + +if test X${enable_m64_build} = Xyes; then + CFLAGS="-m64 $CFLAGS" + LDFLAGS="-m64 $LDFLAGS" +fi +if test X${enable_m32_build} = Xyes; then + CFLAGS="-m32 $CFLAGS" + LDFLAGS="-m32 $LDFLAGS" +fi + AC_CHECK_LIB(m, sin) #-------------------------------------------------------------------- @@ -132,21 +161,56 @@ case $system in AC_CHECK_HEADER(dld.h, [ SHLIB_LD="ld" SHLIB_LDFLAGS="-shared"]) + if test X${enable_m64_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) + fi + if test X${enable_m32_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) + fi fi SHLIB_EXTRACT_ALL="" ;; - *-netbsd*|*-freebsd*|*-openbsd*|*-dragonfly*) + *-openbsd*) + # Not available on all versions: check for include file. + AC_CHECK_HEADER(dlfcn.h, [ + SHLIB_CFLAGS="-fpic" + SHLIB_LD="${CC}" + SHLIB_LDFLAGS="$LDFLAGS -shared" + SHLIB_SUFFIX=".so" + if test X${enable_m64_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) + fi + if test X${enable_m32_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) + fi + ], [ + # No dynamic loading. + SHLIB_CFLAGS="" + SHLIB_LD="ld" + SHLIB_LDFLAGS="" + SHLIB_SUFFIX="" + AC_MSG_ERROR(don't know how to compile and link dynamic drivers) + ]) + SHLIB_EXTRACT_ALL="" + ;; + *-netbsd*|*-freebsd*|*-dragonfly*) # Not available on all versions: check for include file. AC_CHECK_HEADER(dlfcn.h, [ SHLIB_CFLAGS="-fpic" SHLIB_LD="ld" SHLIB_LDFLAGS="$LDFLAGS -Bshareable -x" SHLIB_SUFFIX=".so" + if test X${enable_m64_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) + fi + if test X${enable_m32_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) + fi ], [ # No dynamic loading. SHLIB_CFLAGS="" SHLIB_LD="ld" - SHLIB_LDFLAGS="$LDFLAGS" + SHLIB_LDFLAGS="" SHLIB_SUFFIX="" AC_MSG_ERROR(don't know how to compile and link dynamic drivers) ]) @@ -155,7 +219,13 @@ case $system in *-solaris2*|*-sysv4*) SHLIB_CFLAGS="-KPIC" SHLIB_LD="/usr/ccs/bin/ld" - SHLIB_LDFLAGS="$LDFLAGS -G -z text" + SHLIB_LDFLAGS="$no_mXX_LDFLAGS -G -z text" + if test X${enable_m64_build} = Xyes; then + SHLIB_LDFLAGS="-64 $SHLIB_LDFLAGS" + fi + if test X${enable_m32_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) + fi SHLIB_SUFFIX=".so" SHLIB_EXTRACT_ALL="-z allextract" ;; @@ -170,6 +240,12 @@ case $system in SHLIB_CFLAGS="-fPIC" SHLIB_LD="ld" SHLIB_LDFLAGS="$LDFLAGS -shared" + if test X${enable_m64_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 64-bit dynamic drivers) + fi + if test X${enable_m32_build} = Xyes; then + AC_MSG_ERROR(don't know how to link 32-bit dynamic drivers) + fi SHLIB_SUFFIX=".so" SHLIB_EXTRACT_ALL="" ;; diff --git a/lib/test_server/src/erl2html2.erl b/lib/test_server/src/erl2html2.erl index d0f40c47a7..952036502a 100644 --- a/lib/test_server/src/erl2html2.erl +++ b/lib/test_server/src/erl2html2.erl @@ -1,4 +1,3 @@ -%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% diff --git a/lib/test_server/src/test_server.appup.src b/lib/test_server/src/test_server.appup.src index 0fbe5f23f7..42c6fe2e46 100644 --- a/lib/test_server/src/test_server.appup.src +++ b/lib/test_server/src/test_server.appup.src @@ -1 +1,21 @@ -{"%VSN%",[],[]}.
\ No newline at end of file +%% -*- erlang -*- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +{"%VSN%", + [{<<".*">>,[{restart_application, test_server}]}], + [{<<".*">>,[{restart_application, test_server}]}] +}. diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 6ddb2b615f..82672521f7 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -40,7 +40,7 @@ -export([call_crash/3,call_crash/4,call_crash/5]). -export([temp_name/1]). -export([start_node/3, stop_node/1, wait_for_node/1, is_release_available/1]). --export([app_test/1, app_test/2]). +-export([app_test/1, app_test/2, appup_test/1]). -export([is_native/1]). -export([comment/1, make_priv_dir/0]). -export([os_type/0]). @@ -176,8 +176,6 @@ module_names(Beams) -> do_cover_compile(Modules) -> do_cover_compile1(lists:usort(Modules)). % remove duplicates -do_cover_compile1([Dont|Rest]) when Dont=:=cover -> - do_cover_compile1(Rest); do_cover_compile1([M|Rest]) -> case {code:is_sticky(M),code:is_loaded(M)} of {true,_} -> @@ -405,6 +403,7 @@ run_test_case_apply({CaseNum,Mod,Func,Args,Name, ref :: reference(), pid :: pid(), mf :: {atom(),atom()}, + last_known_loc :: term(), status :: tc_status() | 'undefined', ret_val :: term(), comment :: list(char()), @@ -436,8 +435,9 @@ run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> LogOpts, TCCallback) end), put(test_server_detected_fail, []), - St = #st{ref=Ref,pid=Pid,mf={Mod,Func},status=starting,ret_val=[], - comment="",timeout=infinity,config=hd(Args)}, + St = #st{ref=Ref,pid=Pid,mf={Mod,Func},last_known_loc=unknown, + status=starting,ret_val=[],comment="",timeout=infinity, + config=hd(Args)}, run_test_case_msgloop(St). %% Ugly bug (pre R5A): @@ -537,7 +537,22 @@ run_test_case_msgloop(#st{ref=Ref,pid=Pid,end_conf_pid=EndConfPid0}=St0) -> St = setup_termination(RetVal, St0#st{config=undefined}), run_test_case_msgloop(St); {'EXIT',Pid,Reason} -> - St = handle_tc_exit(Reason, St0), + %% This exit typically happens when an unknown external process + %% has caused a test case process to terminate (e.g. if a linked + %% process has crashed). + St = + case Reason of + {What,[Loc0={_M,_F,A,[{file,_}|_]}|_]} when + is_integer(A) -> + Loc = rewrite_loc_item(Loc0), + handle_tc_exit(What, St0#st{last_known_loc=[Loc]}); + {What,[Details,Loc0={_M,_F,A,[{file,_}|_]}|_]} when + is_integer(A) -> + Loc = rewrite_loc_item(Loc0), + handle_tc_exit({What,Details}, St0#st{last_known_loc=[Loc]}); + _ -> + handle_tc_exit(Reason, St0) + end, run_test_case_msgloop(St); {EndConfPid0,{call_end_conf,Data,_Result}} -> #st{mf={Mod,Func},config=CurrConf} = St0, @@ -695,7 +710,7 @@ handle_tc_exit(Reason, #st{config=Config,mf={Mod,Func0},pid=Pid, {testcase_aborted=E,AbortReason,Loc0} -> {{E,AbortReason},Loc0}; Other -> - {Other,unknown} + {Other,St#st.last_known_loc} end, Func = case Status of init_per_testcase=F -> {F,Func0}; @@ -837,9 +852,13 @@ spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> spawn_link(FwCall); spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> + Func1 = case Func of + {_InitOrEndPerTC,F} -> F; + F -> F + end, FwCall = fun() -> - case catch fw_error_notify(Mod,Func,[], + case catch fw_error_notify(Mod,Func1,[], Error,Loc) of {'EXIT',FwErrorNotifyErr} -> exit({fw_notify_done,error_notification, @@ -847,8 +866,8 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> _ -> ok end, - Conf = [{tc_status,{failed,timetrap_timeout}}|CurrConf], - case catch do_end_tc_call(Mod,Func, + Conf = [{tc_status,{failed,Error}}|CurrConf], + case catch do_end_tc_call(Mod,Func1, {Pid,Error,[Conf]},Error) of {'EXIT',FwEndTCErr} -> exit({fw_notify_done,end_tc,FwEndTCErr}); @@ -915,6 +934,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, put(test_server_logopts, LogOpts), Where = [{Mod,Func}], put(test_server_loc, Where), + FWInitResult = test_server_sup:framework_call(init_tc,[Mod,Func,Args0], {ok,Args0}), set_tc_state(running), @@ -924,7 +944,7 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback); Error = {error,_Reason} -> NewResult = do_end_tc_call(Mod,Func, {Error,Args0}, - {skip,{failed,Error}}), + {auto_skip,{failed,Error}}), {{0,NewResult},Where,[]}; {fail,Reason} -> Conf = [{tc_status,{failed,Reason}} | hd(Args0)], @@ -935,9 +955,9 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, Skip = {skip,_Reason} -> NewResult = do_end_tc_call(Mod,Func, {Skip,Args0}, Skip), {{0,NewResult},Where,[]}; - {auto_skip,Reason} -> - NewResult = do_end_tc_call(Mod,Func, {{skip,Reason},Args0}, - {skip,Reason}), + AutoSkip = {auto_skip,_Reason} -> + %% special case where a conf case "pretends" to be skipped + NewResult = do_end_tc_call(Mod,Func, {AutoSkip,Args0}, AutoSkip), {{0,NewResult},Where,[]} end, exit({Ref,Time,Value,Loc,Opts}). @@ -955,7 +975,8 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> {{0,NewRes},Line,[]}; {skip_and_save,Reason,SaveCfg} -> Line = get_loc(), - Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}|hd(Args)], + Conf = [{tc_status,{skipped,Reason}}, + {save_config,SaveCfg}|hd(Args)], NewRes = do_end_tc_call(Mod,Func, {{skip,Reason},[Conf]}, {skip,Reason}), {{0,NewRes},Line,[]}; @@ -2427,8 +2448,11 @@ app_test(App) -> app_test(App, Mode) -> test_server_sup:app_test(App, Mode). -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% appup_test/1 +%% +appup_test(App) -> + test_server_sup:appup_test(App). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% is_native(Mod) -> true | false diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index d0f31af198..5fbc47a813 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -487,6 +487,7 @@ init([]) -> ok end, test_server_sup:cleanup_crash_dumps(), + test_server_sup:util_start(), State = #state{jobs=[],finish=false}, TI0 = test_server:init_target_info(), TargetHost = test_server_sup:hoststr(), @@ -1055,6 +1056,7 @@ handle_info(_, State) -> %% test suites (if any) and any possible remainting slave node terminate(_Reason, State) -> + test_server_sup:util_stop(), case State#state.trc of false -> ok; Sock -> test_server_node:stop_tracer_node(Sock) @@ -1161,8 +1163,11 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, end, {SkippedN,SkipStr} = case get(test_server_skipped) of - {0,_} -> {0,""}; - {Skipped,_} -> {Skipped,io_lib:format(", ~w Skipped", [Skipped])} + {0,0} -> + {0,""}; + {USkipped,ASkipped} -> + Skipped = USkipped+ASkipped, + {Skipped,io_lib:format(", ~w Skipped", [Skipped])} end, OkN = get(test_server_ok), FailedN = get(test_server_failed), @@ -1402,10 +1407,23 @@ remove_conf([{conf, _Ref, Props, _MF}|Cases], NoConf, Repeats) -> end; remove_conf([{make,_Ref,_MF}|Cases], NoConf, Repeats) -> remove_conf(Cases, NoConf, Repeats); +remove_conf([{skip_case,{{_M,all},_Cmt},_Mode}|Cases], NoConf, Repeats) -> + remove_conf(Cases, NoConf, Repeats); remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt}}|Cases], NoConf, Repeats) when Type==conf; Type==make -> remove_conf(Cases, NoConf, Repeats); +remove_conf([{skip_case,{Type,_Ref,_MF,_Cmt},_Mode}|Cases], + NoConf, Repeats) when Type==conf; + Type==make -> + remove_conf(Cases, NoConf, Repeats); +remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) -> + FwMod = get_fw_mod(?MODULE), + if Mod == FwMod -> + remove_conf(Cases, NoConf, Repeats); + true -> + remove_conf(Cases, [C|NoConf], Repeats) + end; remove_conf([C|Cases], NoConf, Repeats) -> remove_conf(Cases, [C|NoConf], Repeats); remove_conf([], NoConf, true) -> @@ -1413,6 +1431,11 @@ remove_conf([], NoConf, true) -> remove_conf([], NoConf, false) -> lists:reverse(NoConf). +get_suites([{skip_case,{{Mod,_F},_Cmt},_Mode}|Tests], Mods) when is_atom(Mod) -> + case add_mod(Mod, Mods) of + true -> get_suites(Tests, [Mod|Mods]); + false -> get_suites(Tests, Mods) + end; get_suites([{Mod,_Case}|Tests], Mods) when is_atom(Mod) -> case add_mod(Mod, Mods) of true -> get_suites(Tests, [Mod|Mods]); @@ -1478,8 +1501,10 @@ do_test_cases(TopCases, SkipCases, end, put(test_server_cases, N), put(test_server_case_num, 0), + TestSpec = add_init_and_end_per_suite(TestSpec0, undefined, undefined, FwMod), + TI = get_target_info(), print(1, "Starting test~ts", [print_if_known(N, {", ~w test cases",[N]}, @@ -1702,30 +1727,33 @@ make_html_link(LinkName, Target, Explanation) -> ok = write_html_file(LinkName, H). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% start_minor_log_file(Mod, Func) -> AbsName +%% start_minor_log_file(Mod, Func, ParallelTC) -> AbsName %% Mod = atom() %% Func = atom() +%% ParallelTC = bool() %% AbsName = string() %% %% Create a minor log file for the test case Mod,Func,Args. The log file -%% will be stored in the log directory under the name <Mod>.<Func>.log. -%% Some header info will also be inserted into the log file. +%% will be stored in the log directory under the name <Mod>.<Func>.html. +%% Some header info will also be inserted into the log file. If the test +%% case runs in a parallel group, then to avoid clashing file names if the +%% case is executed more than once, the name <Mod>.<Func>.<Timestamp>.html +%% is used. -start_minor_log_file(Mod, Func) -> +start_minor_log_file(Mod, Func, ParallelTC) -> MFA = {Mod,Func,1}, LogDir = get(test_server_log_dir_base), Name0 = lists:flatten(io_lib:format("~w.~w~ts", [Mod,Func,?html_ext])), Name = downcase(Name0), AbsName = filename:join(LogDir, Name), - case file:read_file_info(AbsName) of - {error,_} -> %% normal case, unique name + case (ParallelTC orelse (element(1,file:read_file_info(AbsName))==ok)) of + false -> %% normal case, unique name start_minor_log_file1(Mod, Func, LogDir, AbsName, MFA); - {ok,_} -> %% special case, duplicate names - {_,S,Us} = now(), + true -> %% special case, duplicate names + Tag = test_server_sup:unique_name(), Name1_0 = - lists:flatten(io_lib:format("~w.~w.~w.~w~ts", [Mod,Func,S, - trunc(Us/1000), - ?html_ext])), + lists:flatten(io_lib:format("~w.~w.~ts~ts", [Mod,Func,Tag, + ?html_ext])), Name1 = downcase(Name1_0), AbsName1 = filename:join(LogDir, Name1), start_minor_log_file1(Mod, Func, LogDir, AbsName1, MFA) @@ -1795,23 +1823,40 @@ downcase([], Result) -> %% %% Errors are silently ignored. -html_convert_modules(TestSpec, _Config) -> - Mods = html_isolate_modules(TestSpec), +html_convert_modules(TestSpec, _Config, FwMod) -> + Mods = html_isolate_modules(TestSpec, FwMod), html_convert_modules(Mods), copy_html_files(get(test_server_dir), get(test_server_log_dir_base)). %% Retrieve a list of modules out of the test spec. -html_isolate_modules(List) -> html_isolate_modules(List, sets:new()). - -html_isolate_modules([], Set) -> sets:to_list(Set); -html_isolate_modules([{skip_case,_}|Cases], Set) -> - html_isolate_modules(Cases, Set); -html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set) -> - html_isolate_modules(Cases, sets:add_element(Mod, Set)); -html_isolate_modules([{Mod,_Case}|Cases], Set) -> - html_isolate_modules(Cases, sets:add_element(Mod, Set)); -html_isolate_modules([{Mod,_Case,_Args}|Cases], Set) -> - html_isolate_modules(Cases, sets:add_element(Mod, Set)). +html_isolate_modules(List, FwMod) -> + html_isolate_modules(List, sets:new(), FwMod). + +html_isolate_modules([], Set, _) -> sets:to_list(Set); +html_isolate_modules([{skip_case,{_Case,_Cmt},_Mode}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, Set, FwMod); +html_isolate_modules([{conf,_Ref,Props,{FwMod,_Func}}|Cases], Set, FwMod) -> + Set1 = case proplists:get_value(suite, Props) of + undefined -> Set; + Mod -> sets:add_element(Mod, Set) + end, + html_isolate_modules(Cases, Set1, FwMod); +html_isolate_modules([{conf,_Ref,_Props,{Mod,_Func}}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod); +html_isolate_modules([{skip_case,{conf,_Ref,{FwMod,_Func},_Cmt},Mode}|Cases], + Set, FwMod) -> + Set1 = case proplists:get_value(suite, get_props(Mode)) of + undefined -> Set; + Mod -> sets:add_element(Mod, Set) + end, + html_isolate_modules(Cases, Set1, FwMod); +html_isolate_modules([{skip_case,{conf,_Ref,{Mod,_Func},_Cmt},_Props}|Cases], + Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod); +html_isolate_modules([{Mod,_Case}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod); +html_isolate_modules([{Mod,_Case,_Args}|Cases], Set, FwMod) -> + html_isolate_modules(Cases, sets:add_element(Mod, Set), FwMod). %% Given a list of modules, convert each module's source code to HTML. html_convert_modules([Mod|Mods]) -> @@ -1892,23 +1937,30 @@ copy_html_file(Src, DestDir) -> add_init_and_end_per_suite([{make,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,{{Mod,all},_}}=Case|Cases], LastMod, +add_init_and_end_per_suite([{skip_case,{{Mod,all},_},_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,{{Mod,_},_}}=Case|Cases], LastMod, - LastRef, FwMod) when Mod =/= LastMod -> + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{{Mod,_},_Cmt},_Mode}=Case|Cases], + LastMod, LastRef, FwMod) when Mod =/= LastMod -> + {PreCases, NextMod, NextRef} = + do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_},_}=Case|Cases], + LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([{skip_case,{conf,_,{Mod,_},_}}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; -add_init_and_end_per_suite([{skip_case,_}=Case|Cases], LastMod, LastRef, FwMod) -> - [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, LastRef, FwMod) -> %% if Mod == FwMod, this conf test is (probably) a test case group where @@ -1918,7 +1970,8 @@ add_init_and_end_per_suite([{conf,Ref,Props,{FwMod,Func}}=Case|Cases], LastMod, Suite when Suite =/= undefined, Suite =/= LastMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Suite, FwMod), - Case1 = {conf,Ref,proplists:delete(suite,Props),{FwMod,Func}}, + Case1 = {conf,Ref,[{suite,NextMod}|proplists:delete(suite,Props)], + {FwMod,Func}}, PreCases ++ [Case1|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; _ -> @@ -1928,19 +1981,25 @@ add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; +add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod) + when element(1,SkipCase) == skip_case -> + [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([{Mod,_,_}=Case|Cases], LastMod, LastRef, FwMod) when Mod =/= LastMod, Mod =/= FwMod -> {PreCases, NextMod, NextRef} = do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod), - PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; + PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, + NextRef, FwMod)]; add_init_and_end_per_suite([Case|Cases], LastMod, LastRef, FwMod)-> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([], _LastMod, undefined, _FwMod) -> @@ -2043,7 +2102,8 @@ run_test_cases(TestSpec, Config, TimetrapData) -> true -> ok; false -> - html_convert_modules(TestSpec, Config) + FwMod = get_fw_mod(?MODULE), + html_convert_modules(TestSpec, Config, FwMod) end, run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []), @@ -2116,7 +2176,7 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% comment (which gets printed in the log files) describes why the case %% was skipped. %% -%% {skip_case,{Case,Comment}} A normal test case skipped by the user. +%% {skip_case,{Case,Comment},Mode} A normal test case skipped by the user. %% The comment (which gets printed in the log files) describes why the %% case was skipped. %% @@ -2209,34 +2269,50 @@ run_test_cases(TestSpec, Config, TimetrapData) -> %% group1_end | ---> %% -run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], - Config, TimetrapData, Mode, Status) when Type==conf; - Type==make -> +run_test_cases_loop([{SkipTag,CaseData={Type,_Ref,_Case,_Comment}}|Cases], + Config, TimetrapData, Mode, Status) when + ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and + ((Type==conf) or (Type==make)) -> + run_test_cases_loop([{SkipTag,CaseData,Mode}|Cases], + Config, TimetrapData, Mode, Status); + +run_test_cases_loop([{SkipTag,{Type,Ref,Case,Comment},SkipMode}|Cases], + Config, TimetrapData, Mode, Status) when + ((SkipTag==auto_skip_case) or (SkipTag==skip_case)) and + ((Type==conf) or (Type==make)) -> file:set_cwd(filename:dirname(get(test_server_dir))), CurrIOHandler = get(test_server_common_io_handler), ParentMode = tl(Mode), + {AutoOrUser,ReportTag} = + if SkipTag == auto_skip_case -> {auto,tc_auto_skip}; + SkipTag == skip_case -> {user,tc_user_skip} + end, + %% check and update the mode for test case execution and io msg handling case {curr_ref(Mode),check_props(parallel, Mode)} of {Ref,Ref} -> case check_props(parallel, ParentMode) of false -> - %% this is a skipped end conf for a top level parallel group, - %% buffered io can be flushed + %% this is a skipped end conf for a top level parallel + %% group, buffered io can be flushed handle_test_case_io_and_status(), set_io_buffering(undefined), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, - {Mod,Func,Comment}]), + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, + false, SkipMode), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, + [ReportTag,ConfData]), run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, delete_status(Ref, Status)); _ -> - %% this is a skipped end conf for a parallel group nested under a - %% parallel group (io buffering is active) + %% this is a skipped end conf for a parallel group nested + %% under a parallel group (io buffering is active) wait_for_cases(Ref), - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip, - {Mod,Func,Comment}]), + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, + true, SkipMode), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2246,18 +2322,21 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], _ -> ok end, - run_test_cases_loop(Cases, Config, TimetrapData, ParentMode, + run_test_cases_loop(Cases, Config, + TimetrapData, ParentMode, delete_status(Ref, Status)) end; {Ref,false} -> %% this is a skipped end conf for a non-parallel group that's not %% nested under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), - - %% Check if this group is auto skipped because of error in the init conf. - %% If so, check if the parent group is a sequence, and if it is, skip - %% all proceeding tests in that group. + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, + false, SkipMode), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), + + %% Check if this group is auto skipped because of error in the + %% init conf. If so, check if the parent group is a sequence, + %% and if it is, skip all proceeding tests in that group. GrName = get_name(Mode), Cases1 = case get_tc_results(Status) of @@ -2270,7 +2349,8 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], ParentRef -> Reason = {group_result,GrName,failed}, skip_cases_upto(ParentRef, Cases, - Reason, tc, Mode) + Reason, tc, ParentMode, + SkipTag) end; false -> Cases @@ -2283,8 +2363,10 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {Ref,_} -> %% this is a skipped end conf for a non-parallel group nested under %% a parallel group (io buffering is active) - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, + true, SkipMode), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2299,51 +2381,54 @@ run_test_cases_loop([{auto_skip_case,{Type,Ref,Case,Comment},SkipMode}|Cases], {_,false} -> %% this is a skipped start conf for a group which is not nested %% under a parallel group - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, false, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status); + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, + false, SkipMode), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), + run_test_cases_loop(Cases, Config, TimetrapData, + [conf(Ref,[])|Mode], Status); {_,Ref0} when is_reference(Ref0) -> - %% this is a skipped start conf for a group nested under a parallel group - %% and if this is the first nested group, io buffering must be activated + %% this is a skipped start conf for a group nested under a parallel + %% group and if this is the first nested group, io buffering must + %% be activated if CurrIOHandler == undefined -> set_io_buffering({Ref,self()}); true -> ok end, - {Mod,Func} = skip_case(auto, Ref, 0, Case, Comment, true, SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config, TimetrapData, [conf(Ref,[])|Mode], Status) + {Mod,Func} = skip_case(AutoOrUser, Ref, 0, Case, Comment, + true, SkipMode), + ConfData = {Mod,{Func,get_name(SkipMode)},Comment}, + test_server_sup:framework_call(report, [ReportTag,ConfData]), + run_test_cases_loop(Cases, Config, TimetrapData, + [conf(Ref,[])|Mode], Status) end; run_test_cases_loop([{auto_skip_case,{Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(auto, undefined, get(test_server_case_num)+1, Case, Comment, is_io_buffered(), SkipMode), - test_server_sup:framework_call(report, [tc_auto_skip,{Mod,Func,Comment}]), + test_server_sup:framework_call(report, [tc_auto_skip, + {Mod,{Func,get_name(SkipMode)}, + Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); -run_test_cases_loop([{skip_case,{conf,Ref,Case,Comment}}|Cases0], +run_test_cases_loop([{skip_case,{{Mod,all}=Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) -> - {Mod,Func} = skip_case(user, Ref, 0, Case, Comment, is_io_buffered()), - {Cases,Config1} = - case curr_ref(Mode) of - Ref -> - %% skipped end conf - {Cases0,tl(Config)}; - _ -> - %% skipped start conf - {skip_cases_upto(Ref, Cases0, Comment, conf, Mode),Config} - end, - test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]), - run_test_cases_loop(Cases, Config1, TimetrapData, Mode, - update_status(skipped, Mod, Func, Status)); + skip_case(user, undefined, 0, Case, Comment, false, SkipMode), + test_server_sup:framework_call(report, [tc_user_skip, + {Mod,{all,get_name(SkipMode)}, + Comment}]), + run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status); -run_test_cases_loop([{skip_case,{Case,Comment}}|Cases], +run_test_cases_loop([{skip_case,{Case,Comment},SkipMode}|Cases], Config, TimetrapData, Mode, Status) -> {Mod,Func} = skip_case(user, undefined, get(test_server_case_num)+1, - Case, Comment, is_io_buffered()), - test_server_sup:framework_call(report, [tc_user_skip,{Mod,Func,Comment}]), + Case, Comment, is_io_buffered(), SkipMode), + test_server_sup:framework_call(report, [tc_user_skip, + {Mod,{Func,get_name(SkipMode)}, + Comment}]), run_test_cases_loop(Cases, Config, TimetrapData, Mode, update_status(skipped, Mod, Func, Status)); @@ -2357,8 +2442,9 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, {Ref,Ref} -> case check_props(parallel, tl(Mode0)) of false -> - %% this is an end conf for a top level parallel group, collect - %% results from the test case processes and calc total time + %% this is an end conf for a top level parallel group, + %% collect results from the test case processes + %% and calc total time OkSkipFail = handle_test_case_io_and_status(), file:set_cwd(filename:dirname(get(test_server_dir))), After = ?now, @@ -2597,7 +2683,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, "~n*** ~w returned bad elements in Config: ~p.~n", [Func,Bad]), Reason = {failed,{Mod,init_per_suite,bad_return}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, + auto_skip_case), set_io_buffering(IOHandler), stop_minor_log_file(), run_test_cases_loop(Cases2, Config, TimetrapData, Mode, @@ -2623,7 +2710,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, print(minor, "~n*** ~w failed.~n" " Skipping all cases.", [Func]), Reason = {failed,{Mod,Func,Fail}}, - {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + {skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, + auto_skip_case), Config, update_status(failed, group_result, get_name(Mode), delete_status(Ref, Status2))}; @@ -2635,14 +2723,37 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, set_io_buffering(IOHandler), stop_minor_log_file(), run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); + + {_,{auto_skip,SkipReason},_} -> + %% this case can only happen if the framework (not the user) + %% decides to skip execution of a conf function + {Cases2,Config1,Status3} = + if StartConf -> + ReportAbortRepeat(auto_skipped), + print(minor, "~n*** ~w auto skipped.~n" + " Skipping all cases.", [Func]), + {skip_cases_upto(Ref, Cases, SkipReason, conf, CurrMode, + auto_skip_case), + Config, + delete_status(Ref, Status2)}; + not StartConf -> + ReportRepeatStop(), + print_conf_time(ConfTime), + {Cases,tl(Config),delete_status(Ref, Status2)} + end, + set_io_buffering(IOHandler), + stop_minor_log_file(), + run_test_cases_loop(Cases2, Config1, TimetrapData, Mode, Status3); + {_,{Skip,Reason},_} when StartConf and ((Skip==skip) or (Skip==skipped)) -> ReportAbortRepeat(skipped), print(minor, "~n*** ~w skipped.~n" " Skipping all cases.", [Func]), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), - Config, TimetrapData, Mode, + run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, + CurrMode, skip_case), + [hd(Config)|Config], TimetrapData, Mode, delete_status(Ref, Status2)); {_,{skip_and_save,Reason,_SavedConfig},_} when StartConf -> ReportAbortRepeat(skipped), @@ -2650,13 +2761,15 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, " Skipping all cases.", [Func]), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), - Config, TimetrapData, Mode, + run_test_cases_loop(skip_cases_upto(Ref, Cases, Reason, conf, + CurrMode, skip_case), + [hd(Config)|Config], TimetrapData, Mode, delete_status(Ref, Status2)); {_,_Other,_} when Func == init_per_suite -> print(minor, "~n*** init_per_suite failed to return a Config list.~n", []), Reason = {failed,{Mod,init_per_suite,bad_return}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode), + Cases2 = skip_cases_upto(Ref, Cases, Reason, conf, CurrMode, + auto_skip_case), set_io_buffering(IOHandler), stop_minor_log_file(), run_test_cases_loop(Cases2, Config, TimetrapData, Mode, @@ -2668,7 +2781,6 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, stop_minor_log_file(), run_test_cases_loop(Cases, [hd(Config)|Config], TimetrapData, Mode, Status2); - {_,_EndConfRetVal,Opts} -> %% Check if return_group_result is set (ok, skipped or failed) and %% if so: @@ -2683,7 +2795,8 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, case {curr_ref(Mode),check_prop(sequence, Mode)} of {ParentRef,ParentRef} -> Reason = {group_result,GrName,failed}, - {skip_cases_upto(ParentRef, Cases, Reason, tc, Mode), + {skip_cases_upto(ParentRef, Cases, Reason, tc, + Mode, auto_skip_case), update_status(failed, group_result, GrName, delete_status(Ref, Status2))}; _ -> @@ -2701,16 +2814,19 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, ReportRepeatStop(), set_io_buffering(IOHandler), stop_minor_log_file(), - run_test_cases_loop(Cases2, tl(Config), TimetrapData, Mode, Status3) + run_test_cases_loop(Cases2, tl(Config), TimetrapData, + Mode, Status3) end; -run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, Mode, Status) -> +run_test_cases_loop([{make,Ref,{Mod,Func,Args}}|Cases0], Config, TimetrapData, + Mode, Status) -> case run_test_case(Ref, 0, Mod, Func, Args, skip_init, TimetrapData) of {_,Why={'EXIT',_},_} -> print(minor, "~n*** ~w failed.~n" " Skipping all cases.", [Func]), Reason = {failed,{Mod,Func,Why}}, - Cases = skip_cases_upto(Ref, Cases0, Reason, conf, Mode), + Cases = skip_cases_upto(Ref, Cases0, Reason, conf, Mode, + auto_skip_case), stop_minor_log_file(), run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status); {_,_Whatever,_} -> @@ -2735,7 +2851,14 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) -> TimetrapData, Mode, Status); run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) -> - Num = put(test_server_case_num, get(test_server_case_num)+1), + {Num,RunInit} = + case FwMod = get_fw_mod(?MODULE) of + Mod when Func == error_in_suite -> + {-1,skip_init}; + _ -> + {put(test_server_case_num, get(test_server_case_num)+1), + run_init} + end, %% check the current execution mode and save info about the case if %% detected that printouts to common log files is handled later @@ -2750,7 +2873,7 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) end, case run_test_case(undefined, Num+1, Mod, Func, Args, - run_init, TimetrapData, Mode) of + RunInit, TimetrapData, Mode) of %% callback to framework module failed, exit immediately {_,{framework_error,{FwMod,FwFunc},Reason},_} -> print(minor, "~n*** ~w failed in ~w. Reason: ~p~n", @@ -2791,7 +2914,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) " Skipping all other cases in sequence.", [Func]), Reason = {failed,{Mod,Func}}, - Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, Mode), + Cases2 = skip_cases_upto(Ref, Cases, Reason, tc, + Mode, auto_skip_case), stop_minor_log_file(), run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1) end @@ -2844,7 +2968,6 @@ get_tc_results([{_,{OkSkipFail,_}} | _Status]) -> get_tc_results([]) -> % in case init_per_suite crashed {[],[],[]}. - conf(Ref, Props) -> {Ref,Props,?now}. @@ -3012,13 +3135,13 @@ cases_to_shuffle(Ref, Cases) -> cases_to_shuffle(Ref, [{conf,Ref,_,_} | _]=Cs, N, Ix) -> % end {N-1,Ix,Cs}; -cases_to_shuffle(Ref, [{skip_case,{_,Ref,_,_}} | _]=Cs, N, Ix) -> % end +cases_to_shuffle(Ref, [{skip_case,{_,Ref,_,_},_} | _]=Cs, N, Ix) -> % end {N-1,Ix,Cs}; cases_to_shuffle(Ref, [{conf,Ref1,_,_}=C | Cs], N, Ix) -> % nested group {Cs1,Rest} = get_subcases(Ref1, Cs, []), cases_to_shuffle(Ref, Rest, N+1, [{N,[C|Cs1]} | Ix]); -cases_to_shuffle(Ref, [{skip_case,{_,Ref1,_,_}}=C | Cs], N, Ix) -> % nested group +cases_to_shuffle(Ref, [{skip_case,{_,Ref1,_,_},_}=C | Cs], N, Ix) -> % nested group {Cs1,Rest} = get_subcases(Ref1, Cs, []), cases_to_shuffle(Ref, Rest, N+1, [{N,[C|Cs1]} | Ix]); @@ -3027,7 +3150,7 @@ cases_to_shuffle(Ref, [C | Cs], N, Ix) -> get_subcases(SubRef, [{conf,SubRef,_,_}=C | Cs], SubCs) -> {lists:reverse([C|SubCs]),Cs}; -get_subcases(SubRef, [{skip_case,{_,SubRef,_,_}}=C | Cs], SubCs) -> +get_subcases(SubRef, [{skip_case,{_,SubRef,_,_},_}=C | Cs], SubCs) -> {lists:reverse([C|SubCs]),Cs}; get_subcases(SubRef, [C|Cs], SubCs) -> get_subcases(SubRef, Cs, [C|SubCs]). @@ -3049,10 +3172,6 @@ random_order(N, {Pos,NewSeed}, IxCases, Shuffled) -> %% SendSync determines if start and finished messages must be sent so %% that the printouts can be buffered and handled in order with io from %% parallel processes. - -skip_case(Type, Ref, CaseNum, Case, Comment, SendSync) -> - skip_case(Type, Ref, CaseNum, Case, Comment, SendSync, []). - skip_case(Type, Ref, CaseNum, Case, Comment, SendSync, Mode) -> MF = {Mod,Func} = case Case of {M,F,_A} -> {M,F}; @@ -3075,13 +3194,27 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> ResultCol = if Type == auto -> ?auto_skip_color; Type == user -> ?user_skip_color end, - - Comment1 = reason_to_string(Comment), - print(major, "~n=case ~w:~w", [Mod,Func]), - print(major, "=started ~s", [lists:flatten(timestamp_get(""))]), - print(major, "=result skipped: ~ts", [Comment1]), - print(2,"*** Skipping test case #~w ~w ***", [CaseNum,{Mod,Func}]), + GroupName = case get_name(Mode) of + undefined -> + ""; + GrName -> + GrName1 = cast_to_list(GrName), + print(major, "=group_props ~p", [[{name,GrName1}]]), + GrName1 + end, + print(major, "=started ~s", [lists:flatten(timestamp_get(""))]), + Comment1 = reason_to_string(Comment), + if Type == auto -> + print(major, "=result auto_skipped: ~ts", [Comment1]); + Type == user -> + print(major, "=result skipped: ~ts", [Comment1]) + end, + if CaseNum == 0 -> + print(2,"*** Skipping ~w ***", [{Mod,Func}]); + true -> + print(2,"*** Skipping test case #~w ~w ***", [CaseNum,{Mod,Func}]) + end, TR = xhtml("<tr valign=\"top\">", ["<tr class=\"",odd_or_even(),"\">"]), GroupName = case get_name(Mode) of undefined -> ""; @@ -3097,6 +3230,7 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> "<td><font color=\"~ts\">SKIPPED</font></td>" "<td>~ts</td></tr>\n", [num2str(CaseNum),fw_name(Mod),GroupName,Func,ResultCol,Comment1]), + if CaseNum > 0 -> {US,AS} = get(test_server_skipped), case Type of @@ -3110,12 +3244,14 @@ skip_case1(Type, CaseNum, Mod, Func, Comment, Mode) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% skip_cases_upto(Ref, Cases, Reason, Origin, Mode) -> Cases1 +%% skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> Cases1 %% +%% SkipType = skip_case | auto_skip_case %% Mark all cases tagged with Ref as skipped. -skip_cases_upto(Ref, Cases, Reason, Origin, Mode) -> - {_,Modified,Rest} = modify_cases_upto(Ref, {skip,Reason,Origin,Mode}, Cases), +skip_cases_upto(Ref, Cases, Reason, Origin, Mode, SkipType) -> + {_,Modified,Rest} = + modify_cases_upto(Ref, {skip,Reason,Origin,Mode,SkipType}, Cases), Modified++Rest. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -3151,6 +3287,7 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) -> %% same ref in the list, if not, this *is* an end conf case case lists:any(fun({_,R,_,_}) when R == Ref -> true; ({_,R,_}) when R == Ref -> true; + ({skip_case,{_,R,_,_},_}) when R == Ref -> true; ({skip_case,{_,R,_,_}}) when R == Ref -> true; (_) -> false end, Cases) of @@ -3161,39 +3298,70 @@ modify_cases_upto(Ref, ModOp, Cases, Orig, Alt) -> end. %% next case is a conf with same ref, must be end conf = we're done -modify_cases_upto1(Ref, {skip,Reason,conf,Mode}, [{conf,Ref,_Props,MF}|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,Reason,conf,Mode,skip_case}, + [{conf,Ref,_Props,MF}|T], Orig, Alt) -> + {Orig,[{skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {skip,Reason,conf,Mode,auto_skip_case}, + [{conf,Ref,_Props,MF}|T], Orig, Alt) -> {Orig,[{auto_skip_case,{conf,Ref,MF,Reason},Mode}|Alt],T}; modify_cases_upto1(Ref, {copy,NewRef}, [{conf,Ref,Props,MF}=C|T], Orig, Alt) -> {[C|Orig],[{conf,NewRef,update_repeat(Props),MF}|Alt],T}; %% we've skipped all remaining cases in a sequence -modify_cases_upto1(Ref, {skip,_,tc,_}, [{conf,Ref,_Props,_MF}|_]=Cs, Orig, Alt) -> +modify_cases_upto1(Ref, {skip,_,tc,_,_}, + [{conf,Ref,_Props,_MF}|_]=Cs, Orig, Alt) -> {Orig,Alt,Cs}; %% next is a make case -modify_cases_upto1(Ref, {skip,Reason,_,Mode}, [{make,Ref,MF}|T], Orig, Alt) -> - {Orig,[{auto_skip_case,{make,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, + [{make,Ref,MF}|T], Orig, Alt) -> + {Orig,[{SkipType,{make,Ref,MF,Reason},Mode}|Alt],T}; modify_cases_upto1(Ref, {copy,NewRef}, [{make,Ref,MF}=M|T], Orig, Alt) -> {[M|Orig],[{make,NewRef,MF}|Alt],T}; %% next case is a user skipped end conf with the same ref = we're done -modify_cases_upto1(Ref, {skip,Reason,_,Mode}, [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) -> - {Orig,[{auto_skip_case,{Type,Ref,MF,Reason},Mode}|Alt],T}; -modify_cases_upto1(Ref, {copy,NewRef}, [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, + [{skip_case,{Type,Ref,MF,_Cmt},_}|T], Orig, Alt) -> + {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}, + [{skip_case,{Type,Ref,MF,_Cmt}}|T], Orig, Alt) -> + {Orig,[{SkipType,{Type,Ref,MF,Reason},Mode}|Alt],T}; +modify_cases_upto1(Ref, {copy,NewRef}, + [{skip_case,{Type,Ref,MF,Cmt},Mode}=C|T], Orig, Alt) -> + {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt},Mode}|Alt],T}; +modify_cases_upto1(Ref, {copy,NewRef}, + [{skip_case,{Type,Ref,MF,Cmt}}=C|T], Orig, Alt) -> {[C|Orig],[{skip_case,{Type,NewRef,MF,Cmt}}|Alt],T}; %% next is a skip_case, could be one test case or 'all' in suite, we must proceed -modify_cases_upto1(Ref, ModOp, [{skip_case,{_F,_Cmt}}=MF|T], Orig, Alt) -> +modify_cases_upto1(Ref, ModOp, [{skip_case,{_F,_Cmt},_Mode}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, ModOp, T, [MF|Orig], [MF|Alt]); %% next is a normal case (possibly in a sequence), mark as skipped, or copy, and proceed -modify_cases_upto1(Ref, {skip,Reason,_,Mode}=Op, [{_M,_F}=MF|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,Reason,_,Mode,skip_case}=Op, + [{_M,_F}=MF|T], Orig, Alt) -> + modify_cases_upto1(Ref, Op, T, Orig, [{skip_case,{MF,Reason},Mode}|Alt]); +modify_cases_upto1(Ref, {skip,Reason,_,Mode,auto_skip_case}=Op, + [{_M,_F}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, Op, T, Orig, [{auto_skip_case,{MF,Reason},Mode}|Alt]); modify_cases_upto1(Ref, CopyOp, [{_M,_F}=MF|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [MF|Orig], [MF|Alt]); +%% next is a conf case, modify the Mode arg to keep track of sub groups +modify_cases_upto1(Ref, {skip,Reason,FType,Mode,SkipType}, + [{conf,OtherRef,Props,_MF}|T], Orig, Alt) -> + case hd(Mode) of + {OtherRef,_,_} -> % end conf + modify_cases_upto1(Ref, {skip,Reason,FType,tl(Mode),SkipType}, + T, Orig, Alt); + _ -> % start conf + Mode1 = [conf(OtherRef,Props)|Mode], + modify_cases_upto1(Ref, {skip,Reason,FType,Mode1,SkipType}, + T, Orig, Alt) + end; + %% next is some other case, ignore or copy -modify_cases_upto1(Ref, {skip,_,_,_}=Op, [_|T], Orig, Alt) -> +modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_Other|T], Orig, Alt) -> modify_cases_upto1(Ref, Op, T, Orig, Alt); modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) -> modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]). @@ -3489,7 +3657,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, TSDir = get(test_server_dir), print(major, "=case ~w:~w", [Mod, Func]), - MinorName = start_minor_log_file(Mod, Func), + MinorName = start_minor_log_file(Mod, Func, self() /= Main), print(minor, "<a name=\"top\"></a>", [], internal_raw), MinorBase = filename:basename(MinorName), print(major, "=logfile ~ts", [filename:basename(MinorName)]), @@ -3518,12 +3686,13 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, update_config(hd(Args), [{priv_dir,PrivDir++"/"}, {tc_logfile,MinorName}]) end, - + GrName = get_name(Mode), test_server_sup:framework_call(report, - [tc_start,{{Mod,Func},MinorName}]), + [tc_start,{{Mod,{Func,GrName}}, + MinorName}]), print_props((RunInit==skip_init), get_props(Mode)), - GroupName = case get_name(Mode) of + GrNameStr = case GrName of undefined -> ""; Name -> cast_to_list(Name) end, @@ -3536,14 +3705,14 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, "<td>" ++ Col0 ++ "~ts" ++ Col1 ++ "</td>" "<td><a href=\"~ts\">~w</a></td>" "<td><a href=\"~ts#top\"><</a> <a href=\"~ts#end\">></a></td>", - [num2str(Num),fw_name(Mod),GroupName,EncMinorBase,Func, + [num2str(Num),fw_name(Mod),GrNameStr,EncMinorBase,Func, EncMinorBase,EncMinorBase]), do_unless_parallel(Main, fun erlang:yield/0), %% run the test case {Result,DetectedFail,ProcsBefore,ProcsAfter} = - run_test_case_apply(Num, Mod, Func, [UpdatedArgs], get_name(Mode), + run_test_case_apply(Num, Mod, Func, [UpdatedArgs], GrName, RunInit, TimetrapData), {Time,RetVal,Loc,Opts,Comment} = case Result of @@ -3562,37 +3731,41 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Status = case {Time,RetVal} of {died,{timetrap_timeout,TimetrapTimeout}} -> - progress(failed, Num, Mod, Func, Loc, + progress(failed, Num, Mod, Func, GrName, Loc, timetrap_timeout, TimetrapTimeout, Comment, Style); {died,Reason} -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); - {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped -> - progress(skip, Num, Mod, Func, Loc, Reason, + {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; + Skip==auto_skip -> + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',_Pid,{Skip,Reason}}} when Skip==skip; Skip==skipped -> - progress(skip, Num, Mod, Func, Loc, Reason, + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',_Pid,Reason}} -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {_,{'EXIT',Reason}} -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); - {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed -> - progress(failed, Num, Mod, Func, Loc, Reason, + {_,{Fail,Reason}} when Fail =:= fail; Fail =:= failed -> + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); - {_, {Skip, Reason}} when Skip==skip; Skip==skipped -> - progress(skip, Num, Mod, Func, Loc, Reason, + {_,Reason={auto_skip,_Why}} -> + progress(skip, Num, Mod, Func, GrName, Loc, Reason, + Time, Comment, Style); + {_,{Skip,Reason}} when Skip==skip; Skip==skipped -> + progress(skip, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style); {Time,RetVal} -> case DetectedFail of [] -> - progress(ok, Num, Mod, Func, Loc, RetVal, + progress(ok, Num, Mod, Func, GrName, Loc, RetVal, Time, Comment, Style); Reason -> - progress(failed, Num, Mod, Func, Loc, Reason, + progress(failed, Num, Mod, Func, GrName, Loc, Reason, Time, Comment, Style) end end, @@ -3697,17 +3870,17 @@ num2str(N) -> integer_to_list(N). %% Note: Strings that are to be written to the minor log must %% be prefixed with "=== " here, or the indentation will be wrong. -progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, +progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, Comment, {St0,St1}) -> - {Reason1,{Color,Ret}} = + {Reason1,{Color,Ret,ReportTag}} = if_auto_skip(Reason, - fun() -> {?auto_skip_color,auto_skip} end, - fun() -> {?user_skip_color,skip} end), - print(major, "=result skipped", []), - print(1, "*** SKIPPED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), - test_server_sup:framework_call(report, [tc_done,{Mod,Func, - {skipped,Reason1}}]), + fun() -> {?auto_skip_color,auto_skip,auto_skipped} end, + fun() -> {?user_skip_color,skip,skipped} end), + print(major, "=result ~w: ~p", [ReportTag,Reason1]), + print(1, "*** SKIPPED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, + {ReportTag,Reason1}}]), ReasonStr = reason_to_string(Reason1), ReasonStr1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ReasonStr,[$\n])]), @@ -3731,13 +3904,13 @@ progress(skip, CaseNum, Mod, Func, Loc, Reason, Time, print(minor, "=== reason = ~ts", [ReasonStr1]), Ret; -progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T, +progress(failed, CaseNum, Mod, Func, GrName, Loc, timetrap_timeout, T, Comment0, {St0,St1}) -> print(major, "=result failed: timeout, ~p", [Loc]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, - [tc_done,{Mod,Func, + [tc_done,{Mod,{Func,GrName}, {failed,timetrap_timeout}}]), FormatLastLoc = test_server_sup:format_loc(get_last_loc(Loc)), ErrorReason = io_lib:format("{timetrap_timeout,~ts}", [FormatLastLoc]), @@ -3757,13 +3930,13 @@ progress(failed, CaseNum, Mod, Func, Loc, timetrap_timeout, T, print(minor, "=== reason = timetrap timeout", []), failed; -progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T, +progress(failed, CaseNum, Mod, Func, GrName, Loc, {testcase_aborted,Reason}, _T, Comment0, {St0,St1}) -> print(major, "=result failed: testcase_aborted, ~p", [Loc]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, - [tc_done,{Mod,Func, + [tc_done,{Mod,{Func,GrName}, {failed,testcase_aborted}}]), FormatLastLoc = test_server_sup:format_loc(get_last_loc(Loc)), ErrorReason = io_lib:format("{testcase_aborted,~ts}", [FormatLastLoc]), @@ -3783,12 +3956,12 @@ progress(failed, CaseNum, Mod, Func, Loc, {testcase_aborted,Reason}, _T, print(minor, "=== reason = {testcase_aborted,~p}", [Reason]), failed; -progress(failed, CaseNum, Mod, Func, unknown, Reason, Time, +progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, Comment0, {St0,St1}) -> print(major, "=result failed: ~p, ~w", [Reason,unknown]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), - test_server_sup:framework_call(report, [tc_done,{Mod,Func, + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {failed,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; true -> "~w" @@ -3819,12 +3992,12 @@ progress(failed, CaseNum, Mod, Func, unknown, Reason, Time, print(minor, "=== reason = " ++ FStr, [FormattedReason]), failed; -progress(failed, CaseNum, Mod, Func, Loc, Reason, Time, +progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, Comment0, {St0,St1}) -> print(major, "=result failed: ~p, ~p", [Reason,Loc]), - print(1, "*** FAILED *** ~ts", - [get_info_str(Func, CaseNum, get(test_server_cases))]), - test_server_sup:framework_call(report, [tc_done,{Mod,Func, + print(1, "*** FAILED ~ts ***", + [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {failed,Reason}}]), TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; true -> "~w" @@ -3846,10 +4019,10 @@ progress(failed, CaseNum, Mod, Func, Loc, Reason, Time, print(minor, "=== reason = " ++ FStr, [FormattedReason]), failed; -progress(ok, _CaseNum, Mod, Func, _Loc, RetVal, Time, +progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, Comment0, {St0,St1}) -> print(minor, "successfully completed test case", []), - test_server_sup:framework_call(report, [tc_done,{Mod,Func,ok}]), + test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]), Comment = case RetVal of {comment,RetComment} -> @@ -3920,24 +4093,25 @@ fw_name(Mod) -> if_auto_skip(Reason={failed,{_,init_per_testcase,_}}, True, _False) -> {Reason,True()}; -if_auto_skip({_T,{skip,Reason={failed,{_,init_per_testcase,_}}},_Opts}, True, _False) -> +if_auto_skip({skip,Reason={failed,{_,init_per_testcase,_}}}, True, _False) -> {Reason,True()}; -if_auto_skip({fw_auto_skip,Reason}, True, _False) -> - {Reason,True()}; -if_auto_skip({_T,{skip,{fw_auto_skip,Reason}},_Opts}, True, _False) -> +if_auto_skip({auto_skip,Reason}, True, _False) -> {Reason,True()}; if_auto_skip(Reason, _True, False) -> {Reason,False()}. -update_skip_counters(RetVal, {US,AS}) -> - {_,Result} = if_auto_skip(RetVal, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), +update_skip_counters({_T,Pat,_Opts}, {US,AS}) -> + {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), + Result; +update_skip_counters(Pat, {US,AS}) -> + {_,Result} = if_auto_skip(Pat, fun() -> {US,AS+1} end, fun() -> {US+1,AS} end), Result. -get_info_str(Func, 0, _Cases) -> - atom_to_list(Func); -get_info_str(_Func, CaseNum, unknown) -> +get_info_str(Mod,Func, 0, _Cases) -> + io_lib:format("~w", [{Mod,Func}]); +get_info_str(_Mod,_Func, CaseNum, unknown) -> "test case " ++ integer_to_list(CaseNum); -get_info_str(_Func, CaseNum, Cases) -> +get_info_str(_Mod,_Func, CaseNum, Cases) -> "test case " ++ integer_to_list(CaseNum) ++ " of " ++ integer_to_list(Cases). @@ -4337,18 +4511,18 @@ update_config(Config, []) -> collect_all_cases(Top, Skip) when is_list(Skip) -> Result = - case collect_cases(Top, #cc{mod=[],skip=Skip}) of + case collect_cases(Top, #cc{mod=[],skip=Skip}, []) of {ok,Cases,_St} -> Cases; Other -> Other end, Result. -collect_cases([], St) -> {ok,[],St}; -collect_cases([Case|Cs0], St0) -> - case collect_cases(Case, St0) of +collect_cases([], St, _) -> {ok,[],St}; +collect_cases([Case|Cs0], St0, Mode) -> + case collect_cases(Case, St0, Mode) of {ok,FlatCases1,St1} -> - case collect_cases(Cs0, St1) of + case collect_cases(Cs0, St1, Mode) of {ok,FlatCases2,St} -> {ok,FlatCases1 ++ FlatCases2,St}; {error,_Reason} = Error -> Error @@ -4357,54 +4531,71 @@ collect_cases([Case|Cs0], St0) -> end; -collect_cases({module,Case}, St) when is_atom(Case), is_atom(St#cc.mod) -> - collect_case({St#cc.mod,Case}, St); -collect_cases({module,Mod,Case}, St) -> - collect_case({Mod,Case}, St); -collect_cases({module,Mod,Case,Args}, St) -> - collect_case({Mod,Case,Args}, St); - -collect_cases({dir,SubDir}, St) -> - collect_files(SubDir, "*_SUITE", St); -collect_cases({dir,SubDir,Pattern}, St) -> - collect_files(SubDir, Pattern++"*", St); - -collect_cases({conf,InitF,CaseList,FinMF}, St) when is_atom(InitF) -> - collect_cases({conf,[],{St#cc.mod,InitF},CaseList,FinMF}, St); -collect_cases({conf,InitMF,CaseList,FinF}, St) when is_atom(FinF) -> - collect_cases({conf,[],InitMF,CaseList,{St#cc.mod,FinF}}, St); -collect_cases({conf,InitMF,CaseList,FinMF}, St0) -> - collect_cases({conf,[],InitMF,CaseList,FinMF}, St0); -collect_cases({conf,Props,InitF,CaseList,FinMF}, St) when is_atom(InitF) -> +collect_cases({module,Case}, St, Mode) when is_atom(Case), is_atom(St#cc.mod) -> + collect_case({St#cc.mod,Case}, St, Mode); +collect_cases({module,Mod,Case}, St, Mode) -> + collect_case({Mod,Case}, St, Mode); +collect_cases({module,Mod,Case,Args}, St, Mode) -> + collect_case({Mod,Case,Args}, St, Mode); + +collect_cases({dir,SubDir}, St, Mode) -> + collect_files(SubDir, "*_SUITE", St, Mode); +collect_cases({dir,SubDir,Pattern}, St, Mode) -> + collect_files(SubDir, Pattern++"*", St, Mode); + +collect_cases({conf,InitF,CaseList,FinMF}, St, Mode) when is_atom(InitF) -> + collect_cases({conf,[],{St#cc.mod,InitF},CaseList,FinMF}, St, Mode); +collect_cases({conf,InitMF,CaseList,FinF}, St, Mode) when is_atom(FinF) -> + collect_cases({conf,[],InitMF,CaseList,{St#cc.mod,FinF}}, St, Mode); +collect_cases({conf,InitMF,CaseList,FinMF}, St0, Mode) -> + collect_cases({conf,[],InitMF,CaseList,FinMF}, St0, Mode); +collect_cases({conf,Props,InitF,CaseList,FinMF}, St, Mode) when is_atom(InitF) -> case init_props(Props) of {error,_} -> {ok,[],St}; Props1 -> - collect_cases({conf,Props1,{St#cc.mod,InitF},CaseList,FinMF}, St) + collect_cases({conf,Props1,{St#cc.mod,InitF},CaseList,FinMF}, + St, Mode) end; -collect_cases({conf,Props,InitMF,CaseList,FinF}, St) when is_atom(FinF) -> +collect_cases({conf,Props,InitMF,CaseList,FinF}, St, Mode) when is_atom(FinF) -> case init_props(Props) of {error,_} -> {ok,[],St}; Props1 -> - collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, St) + collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, + St, Mode) end; -collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> +collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St, Mode) -> case init_props(Props) of {error,_} -> {ok,[],St}; Props1 -> Ref = make_ref(), Skips = St#cc.skip, + Props2 = [{suite,St#cc.mod} | lists:delete(suite,Props1)], + Mode1 = [{Ref,Props2,undefined} | Mode], case in_skip_list({St#cc.mod,Conf}, Skips) of {true,Comment} -> % conf init skipped - {ok,[{skip_case,{conf,Ref,InitMF,Comment}} | + {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode1} | [] ++ [{conf,Ref,[],FinMF}]],St}; {true,Name,Comment} when is_atom(Name) -> % all cases skipped - {ok,[{skip_case,{{St#cc.mod,{group,Name}},Comment}}],St}; + case collect_cases(CaseList, St, Mode1) of + {ok,[],_St} = Empty -> + Empty; + {ok,FlatCases,St1} -> + Cases2Skip = FlatCases ++ [{conf,Ref, + keep_name(Props1), + FinMF}], + Skipped = skip_cases_upto(Ref, Cases2Skip, Comment, + conf, Mode1, skip_case), + {ok,[{skip_case,{conf,Ref,InitMF,Comment},Mode1} | + Skipped],St1}; + {error,_Reason} = Error -> + Error + end; {true,ToSkip,_} when is_list(ToSkip) -> % some cases skipped case collect_cases(CaseList, - St#cc{skip=ToSkip++Skips}) of + St#cc{skip=ToSkip++Skips}, Mode1) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St1} -> @@ -4416,7 +4607,7 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> Error end; false -> - case collect_cases(CaseList, St) of + case collect_cases(CaseList, St, Mode1) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St1} -> @@ -4430,8 +4621,8 @@ collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> end end; -collect_cases({make,InitMFA,CaseList,FinMFA}, St0) -> - case collect_cases(CaseList, St0) of +collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) -> + case collect_cases(CaseList, St0, Mode) of {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St} -> Ref = make_ref(), @@ -4440,62 +4631,62 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0) -> {error,_Reason} = Error -> Error end; -collect_cases({Module, Cases}, St) when is_list(Cases) -> - case (catch collect_case(Cases, St#cc{mod=Module}, [])) of +collect_cases({Module, Cases}, St, Mode) when is_list(Cases) -> + case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of {ok, NewCases, NewSt} -> {ok, NewCases, NewSt}; Other -> {error, Other} end; -collect_cases({_Mod,_Case}=Spec, St) -> - collect_case(Spec, St); +collect_cases({_Mod,_Case}=Spec, St, Mode) -> + collect_case(Spec, St, Mode); -collect_cases({_Mod,_Case,_Args}=Spec, St) -> - collect_case(Spec, St); -collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) -> - collect_case({St#cc.mod,Case}, St); -collect_cases(Other, St) -> +collect_cases({_Mod,_Case,_Args}=Spec, St, Mode) -> + collect_case(Spec, St, Mode); +collect_cases(Case, St, Mode) when is_atom(Case), is_atom(St#cc.mod) -> + collect_case({St#cc.mod,Case}, St, Mode); +collect_cases(Other, St, _Mode) -> {error,{bad_subtest_spec,St#cc.mod,Other}}. -collect_case({Mod,{conf,_,_,_,_}=Conf}, St) -> - collect_case_invoke(Mod, Conf, [], St); +collect_case({Mod,{conf,_,_,_,_}=Conf}, St, Mode) -> + collect_case_invoke(Mod, Conf, [], St, Mode); -collect_case(MFA, St) -> +collect_case(MFA, St, Mode) -> case in_skip_list(MFA, St#cc.skip) of {true,Comment} -> - {ok,[{skip_case,{MFA,Comment}}],St}; + {ok,[{skip_case,{MFA,Comment},Mode}],St}; false -> case MFA of - {Mod,Case} -> collect_case_invoke(Mod, Case, MFA, St); + {Mod,Case} -> collect_case_invoke(Mod, Case, MFA, St, Mode); {_Mod,_Case,_Args} -> {ok,[MFA],St} end end. -collect_case([], St, Acc) -> +collect_case([], St, Acc, _Mode) -> {ok, Acc, St}; -collect_case([Case | Cases], St, Acc) -> - {ok, FlatCases, NewSt} = collect_case({St#cc.mod, Case}, St), - collect_case(Cases, NewSt, Acc ++ FlatCases). +collect_case([Case | Cases], St, Acc, Mode) -> + {ok, FlatCases, NewSt} = collect_case({St#cc.mod, Case}, St, Mode), + collect_case(Cases, NewSt, Acc ++ FlatCases, Mode). -collect_case_invoke(Mod, Case, MFA, St) -> +collect_case_invoke(Mod, Case, MFA, St, Mode) -> case get_fw_mod(undefined) of undefined -> case catch apply(Mod, Case, [suite]) of {'EXIT',_} -> {ok,[MFA],St}; Suite -> - collect_subcases(Mod, Case, MFA, St, Suite) + collect_subcases(Mod, Case, MFA, St, Suite, Mode) end; _ -> Suite = test_server_sup:framework_call(get_suite, [Mod,Case], []), - collect_subcases(Mod, Case, MFA, St, Suite) + collect_subcases(Mod, Case, MFA, St, Suite, Mode) end. -collect_subcases(Mod, Case, MFA, St, Suite) -> +collect_subcases(Mod, Case, MFA, St, Suite, Mode) -> case Suite of [] when Case == all -> {ok,[],St}; [] when element(1, Case) == conf -> {ok,[],St}; @@ -4503,28 +4694,28 @@ collect_subcases(Mod, Case, MFA, St, Suite) -> %%%! --- START Kept for backwards compatibility --- %%%! Requirements are not used {req,ReqList} -> - collect_case_deny(Mod, Case, MFA, ReqList, [], St); + collect_case_deny(Mod, Case, MFA, ReqList, [], St, Mode); {req,ReqList,SubCases} -> - collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St); + collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St, Mode); %%%! --- END Kept for backwards compatibility --- {Skip,Reason} when Skip==skip; Skip==skipped -> - {ok,[{skip_case,{MFA,Reason}}],St}; + {ok,[{skip_case,{MFA,Reason},Mode}],St}; {error,Reason} -> throw(Reason); SubCases -> - collect_case_subcases(Mod, Case, SubCases, St) + collect_case_subcases(Mod, Case, SubCases, St, Mode) end. -collect_case_subcases(Mod, Case, SubCases, St0) -> +collect_case_subcases(Mod, Case, SubCases, St0, Mode) -> OldMod = St0#cc.mod, - case collect_cases(SubCases, St0#cc{mod=Mod}) of + case collect_cases(SubCases, St0#cc{mod=Mod}, Mode) of {ok,FlatCases,St} -> {ok,FlatCases,St#cc{mod=OldMod}}; {error,Reason} -> {error,{{Mod,Case},Reason}} end. -collect_files(Dir, Pattern, St) -> +collect_files(Dir, Pattern, St, Mode) -> {ok,Cwd} = file:get_cwd(), Dir1 = filename:join(Cwd, Dir), Wc = filename:join([Dir1,Pattern++code:objfile_extension()]), @@ -4534,7 +4725,7 @@ collect_files(Dir, Pattern, St) -> {error,{collect_fail,Dir,Pattern}}; Mods0 -> Mods = [{path_to_module(Mod),all} || Mod <- lists:sort(Mods0)], - collect_cases(Mods, St) + collect_cases(Mods, St, Mode) end. path_to_module(Path) when is_list(Path) -> @@ -4544,14 +4735,14 @@ path_to_module(Path) when is_list(Path) -> %% anyway. It should be removed or renamed! list_to_atom(filename:rootname(filename:basename(Path))). -collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St) -> +collect_case_deny(Mod, Case, MFA, ReqList, SubCases, St, Mode) -> case {check_deny(ReqList, St#cc.skip),SubCases} of {{denied,Comment},_SubCases} -> - {ok,[{skip_case,{MFA,Comment}}],St}; + {ok,[{skip_case,{MFA,Comment},Mode}],St}; {granted,[]} -> {ok,[MFA],St}; {granted,SubCases} -> - collect_case_subcases(Mod, Case, SubCases, St) + collect_case_subcases(Mod, Case, SubCases, St, Mode) end. check_deny([Req|Reqs], DenyList) -> @@ -4678,7 +4869,7 @@ start_node(Name, Type, Options) -> case controller_call({start_node,Name,Type,Options}, T) of {{ok,Nodename}, Host, Cmd, Info, Warning} -> format(minor, - "Successfully started node ~w on ~tp with command: ~tp", + "Successfully started node ~w on ~tp with command: ~ts", [Nodename, Host, Cmd]), format(major, "=node_start ~w", [Nodename]), case Info of @@ -4694,7 +4885,7 @@ start_node(Name, Type, Options) -> {ok, Nodename}; {fail,{Ret, Host, Cmd}} -> format(minor, - "Failed to start node ~tp on ~tp with command: ~tp~n" + "Failed to start node ~tp on ~tp with command: ~ts~n" "Reason: ~p", [Name, Host, Cmd, Ret]), {fail,Ret}; @@ -4703,7 +4894,7 @@ start_node(Name, Type, Options) -> Ret; {Ret, Host, Cmd} -> format(minor, - "Failed to start node ~tp on ~tp with command: ~tp~n" + "Failed to start node ~tp on ~tp with command: ~ts~n" "Reason: ~p", [Name, Host, Cmd, Ret]), Ret @@ -5309,10 +5500,16 @@ write_html_file(File,Content) -> %% The 'major' log file, which is a pure text file is also written %% with utf8 encoding open_utf8_file(File) -> - file:open(File,[write,{encoding,utf8}]). + case file:open(File,AllOpts=[write,{encoding,utf8}]) of + {error,Reason} -> {error,{Reason,{File,AllOpts}}}; + Result -> Result + end. open_utf8_file(File,Opts) -> - file:open(File,[{encoding,utf8}|Opts]). + case file:open(File,AllOpts=[{encoding,utf8}|Opts]) of + {error,Reason} -> {error,{Reason,{File,AllOpts}}}; + Result -> Result + end. %% Write a file with specified encoding write_file(File,Content,latin1) -> diff --git a/lib/test_server/src/test_server_node.erl b/lib/test_server/src/test_server_node.erl index 619fd463de..582abb2153 100644 --- a/lib/test_server/src/test_server_node.erl +++ b/lib/test_server/src/test_server_node.erl @@ -82,7 +82,7 @@ start_tracer_node(TraceFile,TI) -> Cookie = TI#target_info.cookie, {ok,LSock} = gen_tcp:listen(0,[binary,{reuseaddr,true},{packet,2}]), {ok,TracePort} = inet:port(LSock), - Prog = pick_erl_program(default), + Prog = quote_progname(pick_erl_program(default)), Cmd = lists:concat([Prog, " -sname tracer -hidden -setcookie ", Cookie, " -s ", ?MODULE, " trc ", TraceFile, " ", TracePort, " ", TI#target_info.os_family]), @@ -312,7 +312,7 @@ start_node_peer(SlaveName, OptList, From, TI) -> FailOnError = start_node_get_option_value(fail_on_error, OptList, true), Pa = TI#target_info.test_server_dir, Prog0 = start_node_get_option_value(erl, OptList, default), - Prog = pick_erl_program(Prog0), + Prog = quote_progname(pick_erl_program(Prog0)), Args = case string:str(SuppliedArgs,"-setcookie") of 0 -> "-setcookie " ++ TI#target_info.cookie ++ " " ++ SuppliedArgs; @@ -589,7 +589,32 @@ pick_erl_program(L) -> {release, S} -> find_release(S); this -> - lib:progname() + cast_to_list(lib:progname()) + end. + +%% This is an attempt to distinguish between spaces in the program +%% path and spaces that separate arguments. The program is quoted to +%% allow spaces in the path. +%% +%% Arguments could exist either if the executable is excplicitly given +%% ({prog,String}) or if the -program switch to beam is used and +%% includes arguments (typically done by cerl in OTP test environment +%% in order to ensure that slave/peer nodes are started with the same +%% emulator and flags as the test node. The return from lib:progname() +%% could then typically be '/<full_path_to>/cerl -gcov'). +quote_progname(Progname) -> + do_quote_progname(string:tokens(Progname," ")). + +do_quote_progname([Prog]) -> + "\""++Prog++"\""; +do_quote_progname([Prog,Arg|Args]) -> + case os:find_executable(Prog) of + false -> + do_quote_progname([Prog++" "++Arg | Args]); + _ -> + %% this one has an executable - we assume the rest are arguments + "\""++Prog++"\""++ + lists:flatten(lists:map(fun(X) -> [" ",X] end, [Arg|Args])) end. random_element(L) -> diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl index 377aa21018..3cfa84a52f 100644 --- a/lib/test_server/src/test_server_sup.erl +++ b/lib/test_server/src/test_server_sup.erl @@ -29,10 +29,13 @@ hostatom/0, hostatom/1, hoststr/0, hoststr/1, framework_call/2,framework_call/3,framework_call/4, format_loc/1, - call_trace/1]). + util_start/0, util_stop/0, unique_name/0, + call_trace/1, + appup_test/1]). -include("test_server_internal.hrl"). -define(crash_dump_tar,"crash_dumps.tar.gz"). -define(src_listing_ext, ".src.html"). +-record(util_state, {starter, latest_name}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% timetrap(Timeout,Scale,Pid) -> Handle @@ -263,6 +266,249 @@ app_check_export_all([Mod|Mods]) -> end end. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% appup_test/1 +%% +%% Checks one applications .appup file for obvious errors. +%% Checks.. +%% * .. syntax +%% * .. that version in app file matches appup file version +%% * .. validity of appup instructions +%% +%% For library application this function checks that the proper +%% 'restart_application' upgrade and downgrade clauses exist. +appup_test(Application) -> + case is_app(Application) of + {ok, AppFile} -> + case is_appup(Application, proplists:get_value(vsn, AppFile)) of + {ok, Up, Down} -> + StartMod = proplists:get_value(mod, AppFile), + Modules = proplists:get_value(modules, AppFile), + do_appup_tests(StartMod, Application, Up, Down, Modules); + Error -> + test_server:fail(Error) + end; + Error -> + test_server:fail(Error) + end. + +is_appup(Application, Version) -> + AppupFile = atom_to_list(Application) ++ ".appup", + AppupPath = filename:join([code:lib_dir(Application), "ebin", AppupFile]), + case file:consult(AppupPath) of + {ok, [{Version, Up, Down}]} when is_list(Up), is_list(Down) -> + {ok, Up, Down}; + _ -> + test_server:format( + minor, + "Application upgrade (.appup) file not found, " + "or it has very bad syntax.~n"), + {error, appup_not_readable} + end. + +do_appup_tests(undefined, Application, Up, Down, _Modules) -> + %% library application + case Up of + [{<<".*">>, [{restart_application, Application}]}] -> + case Down of + [{<<".*">>, [{restart_application, Application}]}] -> + ok; + _ -> + test_server:format( + minor, + "Library application needs restart_application " + "downgrade instruction.~n"), + {error, library_downgrade_instruction_malformed} + end; + _ -> + test_server:format( + minor, + "Library application needs restart_application " + "upgrade instruction.~n"), + {error, library_upgrade_instruction_malformed} + end; +do_appup_tests(_, _Application, Up, Down, Modules) -> + %% normal application + case check_appup_clauses_plausible(Up, up, Modules) of + ok -> + case check_appup_clauses_plausible(Down, down, Modules) of + ok -> + test_server:format(minor, "OK~n"); + Error -> + test_server:format(minor, "ERROR ~p~n", [Error]), + test_server:fail(Error) + end; + Error -> + test_server:format(minor, "ERROR ~p~n", [Error]), + test_server:fail(Error) + end. + +check_appup_clauses_plausible([], _Direction, _Modules) -> + ok; +check_appup_clauses_plausible([{Re, Instrs} | Rest], Direction, Modules) + when is_binary(Re) -> + case re:compile(Re) of + {ok, _} -> + case check_appup_instructions(Instrs, Direction, Modules) of + ok -> + check_appup_clauses_plausible(Rest, Direction, Modules); + Error -> + Error + end; + {error, Error} -> + {error, {version_regex_malformed, Re, Error}} + end; +check_appup_clauses_plausible([{V, Instrs} | Rest], Direction, Modules) + when is_list(V) -> + case check_appup_instructions(Instrs, Direction, Modules) of + ok -> + check_appup_clauses_plausible(Rest, Direction, Modules); + Error -> + Error + end; +check_appup_clauses_plausible(Clause, _Direction, _Modules) -> + {error, {clause_malformed, Clause}}. + +check_appup_instructions(Instrs, Direction, Modules) -> + case check_instructions(Direction, Instrs, Instrs, [], [], Modules) of + {_Good, []} -> + ok; + {_, Bad} -> + {error, {bad_instructions, Bad}} + end. + +check_instructions(_, [], _, Good, Bad, _) -> + {lists:reverse(Good), lists:reverse(Bad)}; +check_instructions(UpDown, [Instr | Rest], All, Good, Bad, Modules) -> + case catch check_instruction(UpDown, Instr, All, Modules) of + ok -> + check_instructions(UpDown, Rest, All, [Instr | Good], Bad, Modules); + {error, Reason} -> + NewBad = [{Instr, Reason} | Bad], + check_instructions(UpDown, Rest, All, Good, NewBad, Modules) + end. + +check_instruction(up, {add_module, Module}, _, Modules) -> + %% A new module is added + check_module(Module, Modules); +check_instruction(down, {add_module, Module}, _, Modules) -> + %% An old module is re-added + case (catch check_module(Module, Modules)) of + {error, {unknown_module, Module, Modules}} -> ok; + ok -> throw({error, {existing_readded_module, Module}}) + end; +check_instruction(_, {load_module, Module}, _, Modules) -> + check_module(Module, Modules); +check_instruction(_, {load_module, Module, DepMods}, _, Modules) -> + check_module(Module, Modules), + check_depend(DepMods); +check_instruction(_, {load_module, Module, Pre, Post, DepMods}, _, Modules) -> + check_module(Module, Modules), + check_depend(DepMods), + check_purge(Pre), + check_purge(Post); +check_instruction(up, {delete_module, Module}, _, Modules) -> + case (catch check_module(Module, Modules)) of + {error, {unknown_module, Module, Modules}} -> + ok; + ok -> + throw({error,{existing_module_deleted, Module}}) + end; +check_instruction(down, {delete_module, Module}, _, Modules) -> + check_module(Module, Modules); +check_instruction(_, {update, Module}, _, Modules) -> + check_module(Module, Modules); +check_instruction(_, {update, Module, supervisor}, _, Modules) -> + check_module(Module, Modules); +check_instruction(_, {update, Module, DepMods}, _, Modules) + when is_list(DepMods) -> + check_module(Module, Modules); +check_instruction(_, {update, Module, Change}, _, Modules) -> + check_module(Module, Modules), + check_change(Change); +check_instruction(_, {update, Module, Change, DepMods}, _, Modules) -> + check_module(Module, Modules), + check_change(Change), + check_depend(DepMods); +check_instruction(_, {update, Module, Change, Pre, Post, DepMods}, _, Modules) -> + check_module(Module, Modules), + check_change(Change), + check_purge(Pre), + check_purge(Post), + check_depend(DepMods); +check_instruction(_, + {update, Module, Timeout, Change, Pre, Post, DepMods}, + _, + Modules) -> + check_module(Module, Modules), + check_timeout(Timeout), + check_change(Change), + check_purge(Pre), + check_purge(Post), + check_depend(DepMods); +check_instruction(_, + {update, Module, ModType, Timeout, Change, Pre, Post, DepMods}, + _, + Modules) -> + check_module(Module, Modules), + check_mod_type(ModType), + check_timeout(Timeout), + check_change(Change), + check_purge(Pre), + check_purge(Post), + check_depend(DepMods); +check_instruction(_, {restart_application, Application}, _, _) -> + check_application(Application); +check_instruction(_, {remove_application, Application}, _, _) -> + check_application(Application); +check_instruction(_, {add_application, Application}, _, _) -> + check_application(Application); +check_instruction(_, {add_application, Application, Type}, _, _) -> + check_application(Application), + check_restart_type(Type); +check_instruction(_, Instr, _, _) -> + throw({error, {low_level_or_invalid_instruction, Instr}}). + +check_module(Module, Modules) when is_atom(Module) -> + case {is_atom(Module), lists:member(Module, Modules)} of + {true, true} -> ok; + {true, false} -> throw({error, {unknown_module, Module}}); + {false, _} -> throw({error, {bad_module, Module}}) + end. + +check_application(App) -> + case is_atom(App) of + true -> ok; + false -> throw({error, {bad_application, App}}) + end. + +check_depend(Dep) when is_list(Dep) -> ok; +check_depend(Dep) -> throw({error, {bad_depend, Dep}}). + +check_restart_type(permanent) -> ok; +check_restart_type(transient) -> ok; +check_restart_type(temporary) -> ok; +check_restart_type(load) -> ok; +check_restart_type(none) -> ok; +check_restart_type(Type) -> throw({error, {bad_restart_type, Type}}). + +check_timeout(T) when is_integer(T), T > 0 -> ok; +check_timeout(default) -> ok; +check_timeout(infinity) -> ok; +check_timeout(T) -> throw({error, {bad_timeout, T}}). + +check_mod_type(static) -> ok; +check_mod_type(dynamic) -> ok; +check_mod_type(Type) -> throw({error, {bad_mod_type, Type}}). + +check_purge(soft_purge) -> ok; +check_purge(brutal_purge) -> ok; +check_purge(Purge) -> throw({error, {bad_purge, Purge}}). + +check_change(soft) -> ok; +check_change({advanced, _}) -> ok; +check_change(Change) -> throw({error, {bad_change, Change}}). + %% Given two sorted lists, L1 and L2, returns {NotInL2, NotInL1}, %% NotInL2 is the elements of L1 which don't occurr in L2, %% NotInL1 is the elements of L2 which don't ocurr in L1. @@ -583,6 +829,69 @@ downcase([], Result) -> lists:reverse(Result). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% util_start() -> ok +%% +%% Start local utility process +util_start() -> + Starter = self(), + case whereis(?MODULE) of + undefined -> + spawn_link(fun() -> + register(?MODULE, self()), + util_loop(#util_state{starter=Starter}) + end); + _Pid -> + ok + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% util_stop() -> ok +%% +%% Stop local utility process +util_stop() -> + try (?MODULE ! {self(),stop}) of + _ -> + receive {?MODULE,stopped} -> ok + after 5000 -> exit(whereis(?MODULE), kill) + end + catch + _:_ -> + ok + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% unique_name() -> string() +%% +unique_name() -> + ?MODULE ! {self(),unique_name}, + receive {?MODULE,Name} -> Name + after 5000 -> exit({?MODULE,no_util_process}) + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% util_loop(State) -> ok +%% +util_loop(State) -> + receive + {From,unique_name} -> + {_,S,Us} = now(), + Ms = trunc(Us/1000), + Name = lists:flatten(io_lib:format("~w.~w", [S,Ms])), + if Name == State#util_state.latest_name -> + timer:sleep(1), + self() ! {From,unique_name}, + util_loop(State); + true -> + From ! {?MODULE,Name}, + util_loop(State#util_state{latest_name = Name}) + end; + {From,stop} -> + catch unlink(State#util_state.starter), + From ! {?MODULE,stopped}, + ok + end. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% call_trace(TraceSpecFile) -> ok %% %% Read terms on format {m,Mod} | {f,Mod,Func} diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index 189a71a8ce..bc7d244c7c 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -24,7 +24,7 @@ -module(ts). --export([run/0, run/1, run/2, run/3, run/4, +-export([run/0, run/1, run/2, run/3, run/4, run/5, tests/0, tests/1, install/0, install/1, bench/0, bench/1, bench/2, benchmarks/0, @@ -212,6 +212,12 @@ run_all(_Vars) -> run_some([], _Opts) -> ok; +run_some([{Spec,Mod}|Specs], Opts) -> + case run(Spec, Mod, Opts) of + ok -> ok; + Error -> io:format("~p: ~p~n",[{Spec,Mod},Error]) + end, + run_some(Specs, Opts); run_some([Spec|Specs], Opts) -> case run(Spec, Opts) of ok -> ok; @@ -263,8 +269,17 @@ run(List, Opts) when is_list(List), is_list(Opts) -> run_some(List, Opts); %% run/2 -%% Runs one test spec with Options -run(Testspec, Config) when is_atom(Testspec), is_list(Config) -> +%% Runs one test spec with list of suites or with options +run(Testspec, ModsOrConfig) when is_atom(Testspec), + is_list(ModsOrConfig) -> + case is_list_of_suites(ModsOrConfig) of + false -> + run(Testspec, {config_list,ModsOrConfig}); + true -> + run_some([{Testspec,M} || M <- ModsOrConfig], + [batch]) + end; +run(Testspec, {config_list,Config}) -> Options=check_test_get_opts(Testspec, Config), IsSmoke=proplists:get_value(smoke,Config), File=atom_to_list(Testspec), @@ -310,34 +325,95 @@ run(Testspec, Config) when is_atom(Testspec), is_list(Config) -> run_test(File, [{spec,[Spec]}], Options); %% Runs one module in a spec (interactive) run(Testspec, Mod) when is_atom(Testspec), is_atom(Mod) -> - run_test({atom_to_list(Testspec), Mod}, + run_test({atom_to_list(Testspec),Mod}, [{suite,Mod}], [interactive]). %% run/3 %% Run one module in a spec with Config -run(Testspec,Mod,Config) when is_atom(Testspec), is_atom(Mod), is_list(Config) -> +run(Testspec, Mod, Config) when is_atom(Testspec), + is_atom(Mod), + is_list(Config) -> Options=check_test_get_opts(Testspec, Config), - run_test({atom_to_list(Testspec), Mod}, - [{suite,Mod}], - Options); - -%% Runs one testcase in a module. -run(Testspec, Mod, Case) when is_atom(Testspec), is_atom(Mod), is_atom(Case) -> + run_test({atom_to_list(Testspec),Mod}, + [{suite,Mod}], Options); +%% Run multiple modules with Config +run(Testspec, Mods, Config) when is_atom(Testspec), + is_list(Mods), + is_list(Config) -> + run_some([{Testspec,M} || M <- Mods], Config); +%% Runs one test case in a module. +run(Testspec, Mod, Case) when is_atom(Testspec), + is_atom(Mod), + is_atom(Case) -> + Options=check_test_get_opts(Testspec, []), + Args = [{suite,Mod},{testcase,Case}], + run_test(atom_to_list(Testspec), Args, Options); +%% Runs one or more groups in a module. +run(Testspec, Mod, Grs={group,_Groups}) when is_atom(Testspec), + is_atom(Mod) -> Options=check_test_get_opts(Testspec, []), - Args = [{suite,atom_to_list(Mod)},{testcase,atom_to_list(Case)}], + Args = [{suite,Mod},Grs], + run_test(atom_to_list(Testspec), Args, Options); +%% Runs one or more test cases in a module. +run(Testspec, Mod, TCs={testcase,_Cases}) when is_atom(Testspec), + is_atom(Mod) -> + Options=check_test_get_opts(Testspec, []), + Args = [{suite,Mod},TCs], run_test(atom_to_list(Testspec), Args, Options). %% run/4 -%% Run one testcase in a module with Options. +%% Run one test case in a module with Options. run(Testspec, Mod, Case, Config) when is_atom(Testspec), is_atom(Mod), is_atom(Case), is_list(Config) -> Options=check_test_get_opts(Testspec, Config), - Args = [{suite,atom_to_list(Mod)}, {testcase,atom_to_list(Case)}], + Args = [{suite,Mod},{testcase,Case}], + run_test(atom_to_list(Testspec), Args, Options); +%% Run one or more test cases in a module with Options. +run(Testspec, Mod, {testcase,Cases}, Config) when is_atom(Testspec), + is_atom(Mod) -> + run(Testspec, Mod, Cases, Config); +run(Testspec, Mod, Cases, Config) when is_atom(Testspec), + is_atom(Mod), + is_list(Cases), + is_list(Config) -> + Options=check_test_get_opts(Testspec, Config), + Args = [{suite,Mod},Cases], + run_test(atom_to_list(Testspec), Args, Options); +%% Run one or more groups in a module with Options. +run(Testspec, Mod, Grs={group,_Groups}, Config) when is_atom(Testspec), + is_atom(Mod) -> + Options=check_test_get_opts(Testspec, Config), + Args = [{suite,Mod},Grs], run_test(atom_to_list(Testspec), Args, Options). +%% run/5 +%% Run one or more test cases in a group with Options. +run(Testspec, Mod, Group, Cases, Config) when is_atom(Testspec), + is_atom(Mod), + is_list(Config) -> + Group1 = if is_tuple(Group) -> Group; true -> {group,Group} end, + Cases1 = if is_tuple(Cases) -> Cases; true -> {testcase,Cases} end, + Options=check_test_get_opts(Testspec, Config), + Args = [{suite,Mod},Group1,Cases1], + run_test(atom_to_list(Testspec), Args, Options). + +is_list_of_suites(List) -> + lists:all(fun(Suite) -> + S = if is_atom(Suite) -> atom_to_list(Suite); + true -> Suite + end, + try lists:last(string:tokens(S,"_")) of + "SUITE" -> true; + "suite" -> true; + _ -> false + catch + _:_ -> false + end + end, List). + %% Create a spec to skip all SUITES, this is used when the application %% to be tested is not part of the OTP release to be tested. create_skip_spec(Testspec, SuitesToSkip) -> diff --git a/lib/test_server/src/ts.unix.config b/lib/test_server/src/ts.unix.config index 5a2580f464..a34857b9e5 100644 --- a/lib/test_server/src/ts.unix.config +++ b/lib/test_server/src/ts.unix.config @@ -2,3 +2,5 @@ %% Always run a (VNC) X server on host %% {xserver, "xserver.example.com:66"}. + +{unix,[{telnet,"belegost"},{username,"bofh"},{password,"root"},{keep_alive,true}]}. diff --git a/lib/test_server/src/ts_erl_config.erl b/lib/test_server/src/ts_erl_config.erl index 8cd6a333d8..2d45d39700 100644 --- a/lib/test_server/src/ts_erl_config.erl +++ b/lib/test_server/src/ts_erl_config.erl @@ -109,7 +109,10 @@ erts_lib(Vars,OsType) -> ErtsLibIncludeInternal, ErtsLibIncludeInternalGenerated, ErtsLibPath, - ErtsLibInternalPath} + ErtsLibInternalPath, + ErtsLibEthreadMake, + ErtsLibInternalMake + } = case erl_root(Vars) of {installed, _Root} -> Erts = lib_dir(Vars, erts), @@ -117,12 +120,17 @@ erts_lib(Vars,OsType) -> ErtsIncludeInternal = filename:join([ErtsInclude, "internal"]), ErtsLib = filename:join([Erts, "lib"]), ErtsLibInternal = filename:join([ErtsLib, "internal"]), + ErtsEthreadMake = filename:join([ErtsIncludeInternal, "ethread.mk"]), + ErtsInternalMake = filename:join([ErtsIncludeInternal, "erts_internal.mk"]), + {ErtsInclude, ErtsInclude, ErtsIncludeInternal, ErtsIncludeInternal, ErtsLib, - ErtsLibInternal}; + ErtsLibInternal, + ErtsEthreadMake, + ErtsInternalMake}; {srctree, Root, Target} -> Erts = filename:join([Root, "erts"]), ErtsInclude = filename:join([Erts, "include"]), @@ -136,12 +144,17 @@ erts_lib(Vars,OsType) -> "lib", "internal", Target]), + ErtsEthreadMake = filename:join([ErtsIncludeInternalTarget, "ethread.mk"]), + ErtsInternalMake = filename:join([ErtsIncludeInternalTarget, "erts_internal.mk"]), + {ErtsInclude, ErtsIncludeTarget, ErtsIncludeInternal, ErtsIncludeInternalTarget, ErtsLib, - ErtsLibInternal} + ErtsLibInternal, + ErtsEthreadMake, + ErtsInternalMake} end, [{erts_lib_include, quote(filename:nativename(ErtsLibInclude))}, @@ -154,7 +167,9 @@ erts_lib(Vars,OsType) -> {erts_lib_path, quote(filename:nativename(ErtsLibPath))}, {erts_lib_internal_path, quote(filename:nativename(ErtsLibInternalPath))}, {erts_lib_multi_threaded, erts_lib_name(multi_threaded, OsType)}, - {erts_lib_single_threaded, erts_lib_name(single_threaded, OsType)} + {erts_lib_single_threaded, erts_lib_name(single_threaded, OsType)}, + {erts_lib_make_ethread, quote(ErtsLibEthreadMake)}, + {erts_lib_make_internal, quote(ErtsLibInternalMake)} | Vars]. erl_include(Vars) -> @@ -190,10 +205,10 @@ erl_interface(Vars,OsType) -> case erl_root(Vars) of {installed, _Root} -> {filename:join(Dir, "lib"), - filename:join(Dir, "src")}; + filename:join([Dir, "src", "eidefs.mk"])}; {srctree, _Root, Target} -> {filename:join([Dir, "obj", Target]), - filename:join([Dir, "src", Target])} + filename:join([Dir, "src", Target, "eidefs.mk"])} end} end, Lib = link_library("erl_interface",OsType), @@ -234,10 +249,10 @@ erl_interface(Vars,OsType) -> end, [{erl_interface_libpath, quote(filename:nativename(LibPath))}, {erl_interface_sock_libs, sock_libraries(OsType)}, - {erl_interface_lib, Lib}, - {erl_interface_eilib, Lib1}, - {erl_interface_lib_drv, LibDrv}, - {erl_interface_eilib_drv, Lib1Drv}, + {erl_interface_lib, quote(filename:join(LibPath, Lib))}, + {erl_interface_eilib, quote(filename:join(LibPath, Lib1))}, + {erl_interface_lib_drv, quote(filename:join(LibPath, LibDrv))}, + {erl_interface_eilib_drv, quote(filename:join(LibPath, Lib1Drv))}, {erl_interface_threadlib, ThreadLib}, {erl_interface_include, quote(filename:nativename(Incl))}, {erl_interface_mk_include, quote(filename:nativename(MkIncl))} @@ -260,7 +275,7 @@ ic(Vars, OsType) -> end, [{ic_classpath, quote(filename:nativename(ClassPath))}, {ic_libpath, quote(filename:nativename(LibPath))}, - {ic_lib, link_library("ic", OsType)}, + {ic_lib, quote(filename:join(filename:nativename(LibPath),link_library("ic", OsType)))}, {ic_include_path, quote(filename:nativename(Incl))}|Vars]. jinterface(Vars, _OsType) -> @@ -370,9 +385,18 @@ separators(Vars, {win32,_}) -> separators(Vars, _) -> [{'DS',"/"},{'PS',":"}|Vars]. -quote([$ |R]) -> - "\\ "++quote(R); -quote([C|R]) -> - [C|quote(R)]; -quote([]) -> - []. +quote(List) -> + case lists:member($ , List) of + false -> List; + true -> make_quote(List) + end. + +make_quote(List) -> + case os:type() of + {win32, _} -> %% nmake" + [$"] ++ List ++ [$"]; + _ -> %% make + BackQuote = fun($ , Acc) -> [$\\ , $ |Acc]; + (Char, Acc) -> [Char|Acc] end, + lists:foldr(BackQuote, [], List) + end. diff --git a/lib/test_server/src/ts_install.erl b/lib/test_server/src/ts_install.erl index e9e559df5d..bc62015ac3 100644 --- a/lib/test_server/src/ts_install.erl +++ b/lib/test_server/src/ts_install.erl @@ -112,6 +112,12 @@ get_vars([], name, [], Result) -> get_vars(_, _, _, _) -> {error, fatal_bad_conf_vars}. +config_flags() -> + case os:getenv("CONFIG_FLAGS") of + false -> []; + CF -> string:tokens(CF, " \t\n") + end. + unix_autoconf(XConf) -> Configure = filename:absname("configure"), Flags = proplists:get_value(crossflags,XConf,[]), @@ -122,11 +128,14 @@ unix_autoconf(XConf) -> erlang:system_info(threads) /= false], Debug = [" --enable-debug-mode" || string:str(erlang:system_info(system_version),"debug") > 0], - Args = Host ++ Build ++ Threads ++ Debug, + MXX_Build = [Y || Y <- config_flags(), + Y == "--enable-m64-build" + orelse Y == "--enable-m32-build"], + Args = Host ++ Build ++ Threads ++ Debug ++ " " ++ MXX_Build, case filelib:is_file(Configure) of true -> OSXEnv = macosx_cflags(), - io:format("Running ~sEnv: ~p~n", + io:format("Running ~s~nEnv: ~p~n", [lists:flatten(Configure ++ Args),Env++OSXEnv]), Port = open_port({spawn, lists:flatten(["\"",Configure,"\"",Args])}, [stream, eof, {env,Env++OSXEnv}]), @@ -135,7 +144,6 @@ unix_autoconf(XConf) -> {error, no_configure_script} end. - get_xcomp_flag(Flag, Flags) -> get_xcomp_flag(Flag, Flag, Flags). get_xcomp_flag(Flag, Tag, Flags) -> diff --git a/lib/test_server/src/ts_lib.erl b/lib/test_server/src/ts_lib.erl index 52bb346043..5368960446 100644 --- a/lib/test_server/src/ts_lib.erl +++ b/lib/test_server/src/ts_lib.erl @@ -145,16 +145,13 @@ suite_order(sasl) -> 16; suite_order(tools) -> 18; suite_order(runtime_tools) -> 19; suite_order(parsetools) -> 20; -suite_order(pman) -> 21; suite_order(debugger) -> 22; -suite_order(toolbar) -> 23; suite_order(ic) -> 24; suite_order(orber) -> 26; suite_order(inets) -> 28; suite_order(asn1) -> 30; suite_order(os_mon) -> 32; suite_order(snmp) -> 38; -suite_order(mnesia_session) -> 42; suite_order(mnesia) -> 44; suite_order(system) -> 999; %% IMPORTANT: system SHOULD always be last! suite_order(_) -> 200. @@ -167,7 +164,7 @@ subst_file(In, Out, Vars) -> case file:read_file(In) of {ok, Bin} -> Subst = subst(b2s(Bin), Vars, []), - case file:write_file(Out, Subst) of + case file:write_file(Out, unicode:characters_to_binary(Subst)) of ok -> ok; {error, Reason} -> diff --git a/lib/test_server/src/ts_make.erl b/lib/test_server/src/ts_make.erl index f3266f5836..8727f7ebfe 100644 --- a/lib/test_server/src/ts_make.erl +++ b/lib/test_server/src/ts_make.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -67,7 +67,7 @@ get_port_data(Port, Last0, Complete0) -> end. update_last([C|Rest], Line, true) -> - io:put_chars(Line), + io:put_chars(list_to_binary(Line)), %% Utf-8 list to utf-8 binary io:nl(), update_last([C|Rest], [], false); update_last([$\r|Rest], Result, Complete) -> @@ -79,7 +79,7 @@ update_last([C|Rest], Result, Complete) -> update_last([], Result, Complete) -> {Result, Complete}; update_last(eof, Result, _) -> - Result. + unicode:characters_to_list(list_to_binary(Result)). run_make_script({win32, _}, Make, Dir, Makefile) -> {"run_make.bat", diff --git a/lib/test_server/src/ts_run.erl b/lib/test_server/src/ts_run.erl index 60c9a7a4b7..18d021f780 100644 --- a/lib/test_server/src/ts_run.erl +++ b/lib/test_server/src/ts_run.erl @@ -224,7 +224,7 @@ make_command(Vars, Spec, State) -> CrashFile = filename:join(Cwd,State#state.file ++ "_erl_crash.dump"), case filelib:is_file(CrashFile) of true -> - io:format("ts_run: Deleting dump: ~s\n",[CrashFile]), + io:format("ts_run: Deleting dump: ~ts\n",[CrashFile]), file:delete(CrashFile); false -> ok @@ -258,8 +258,8 @@ make_command(Vars, Spec, State) -> run_batch(Vars, _Spec, State) -> process_flag(trap_exit, true), Command = State#state.command ++ " -noinput -s erlang halt", - ts_lib:progress(Vars, 1, "Command: ~s~n", [Command]), - io:format(user, "Command: ~s~n",[Command]), + ts_lib:progress(Vars, 1, "Command: ~ts~n", [Command]), + io:format(user, "Command: ~ts~n",[Command]), Port = open_port({spawn, Command}, [stream, in, eof]), Timeout = 30000 * case os:getenv("TS_RUN_VALGRIND") of false -> 1; @@ -398,8 +398,9 @@ make_common_test_args(Args0, Options0, _Vars) -> end, ConfigFiles = [{config,[filename:join(ConfigPath,File) || File <- get_config_files()]}], - io_lib:format("~100000p",[Args0++Trace++Cover++Logdir++ - ConfigFiles++Options++TimeTrap]). + io_lib:format("~100000p",[[{abort_if_missing_suites,true} | + Args0++Trace++Cover++Logdir++ + ConfigFiles++Options++TimeTrap]]). to_list(X) when is_atom(X) -> atom_to_list(X); diff --git a/lib/test_server/test/test_server_SUITE.erl b/lib/test_server/test/test_server_SUITE.erl index 8ad5fcfb5c..d1772ac411 100644 --- a/lib/test_server/test/test_server_SUITE.erl +++ b/lib/test_server/test/test_server_SUITE.erl @@ -1,4 +1,3 @@ -%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% @@ -117,7 +116,7 @@ test_server_shuffle01_SUITE(Config) -> test_server_skip_SUITE(Config) -> run_test_server_tests("test_server_skip_SUITE", [], - 3, 0, 1, 0, 0, 1, 3, 0, 0, Config). + 3, 0, 1, 0, 1, 0, 3, 0, 0, Config). test_server_conf01_SUITE(Config) -> run_test_server_tests("test_server_conf01_SUITE", [], @@ -248,11 +247,13 @@ run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc, {NActualSkip,NActualFail,NActualSucc} = lists:foldl(fun(#tc{ result = skip },{S,F,Su}) -> {S+1,F,Su}; - (#tc{ result = ok },{S,F,Su}) -> - {S,F,Su+1}; - (#tc{ result = failed },{S,F,Su}) -> - {S,F+1,Su} - end,{0,0,0},Data#suite.cases), + (#tc{ result = auto_skip },{S,F,Su}) -> + {S+1,F,Su}; + (#tc{ result = ok },{S,F,Su}) -> + {S,F,Su+1}; + (#tc{ result = failed },{S,F,Su}) -> + {S,F+1,Su} + end,{0,0,0},Data#suite.cases), Data. translate_filename(Filename,EncodingOnTestNode) -> diff --git a/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl b/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl index 284b51babe..03f371fc8c 100644 --- a/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl +++ b/lib/test_server/test/test_server_SUITE_data/test_server_unicode_SUITE.erl @@ -1,4 +1,3 @@ -%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% diff --git a/lib/test_server/test/test_server_test_lib.erl b/lib/test_server/test/test_server_test_lib.erl index cd6804f7ad..82a702d59f 100644 --- a/lib/test_server/test/test_server_test_lib.erl +++ b/lib/test_server/test/test_server_test_lib.erl @@ -17,6 +17,7 @@ %% %CopyrightEnd% %% -module(test_server_test_lib). + -export([parse_suite/1]). -export([init/2, pre_init_per_testcase/3, post_end_per_testcase/4]). @@ -185,7 +186,9 @@ parse_case("=result" ++ Result, _, Tc) -> "failed" ++ _ -> {ok, Tc#tc{ result = failed } }; "skipped" ++ _ -> - {ok, Tc#tc{ result = skip } } + {ok, Tc#tc{ result = skip } }; + "auto_skipped" ++ _ -> + {ok, Tc#tc{ result = auto_skip } } end; parse_case("=finished" ++ _ , _Fd, #tc{ name = undefined }) -> finished; diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk index 0e37512dc2..6871b5bd14 100644 --- a/lib/test_server/vsn.mk +++ b/lib/test_server/vsn.mk @@ -1 +1 @@ -TEST_SERVER_VSN = 3.6.3 +TEST_SERVER_VSN = 3.6.4 |