Introduce test categories for OTP tests

6 files changed, 138 insertions, 18 deletions

diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 864f82cb63..df60e5f7f2 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -1005,6 +1005,31 @@
 	  for starting the tests, the relaxed scanner
 	  mode is enabled by means of the tuple: <c>{allow_user_terms,true}</c></p>
 	</section>
+	<section>
+	  <title>Reading test specification terms</title>
+	  <p>It's possible to look up terms in the current test specification
+	  (i.e. the spec that's been used to configure and run the current test).
+	  The function <c>get_testspec_terms()</c> returns a list of all test spec
+	  terms (both config- and test terms) and <c>get_testspec_terms(Tags)</c>
+	  returns the term (or a list of terms) matching the tag (or tags) in
+	  <c>Tags</c>.</p>
+	  <p>For example, in the test specification:</p>
+	  <pre>
+	    ...
+	    {label, my_server_smoke_test}.
+	    {config, "../../my_server_setup.cfg"}.
+	    {config, "../../my_server_interface.cfg"}.
+	    ...</pre>	    
+	  <p>And in e.g. a test suite or a CT hook function:</p>
+	  <pre>
+	    ...
+	    [{label,[{_Node,TestType}]}, {config,CfgFiles}] =
+                ct:get_testspec_terms([label,config]),
+
+            [verify_my_server_cfg(TestType, CfgFile) || {Node,CfgFile} &lt;- CfgFiles,
+                                                        Node == node()];
+	    ...</pre>
+	</section>
   </section>
 
   <section>
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 9d8fce2789..5ed1346f1e 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -79,6 +79,7 @@
 %% Other interface functions
 -export([get_status/0, abort_current_testcase/1,
 	 get_event_mgr_ref/0,
+	 get_testspec_terms/0, get_testspec_terms/1,
 	 encrypt_config_file/2, encrypt_config_file/3,
 	 decrypt_config_file/2, decrypt_config_file/3]).
 
@@ -463,6 +464,50 @@ reload_config(Required)->
     ct_config:reload_config(Required).
 
 %%%-----------------------------------------------------------------
+%%% @spec get_testspec_terms() -> TestSpecTerms | undefined
+%%%      TestSpecTerms = [{Tag,Value}]
+%%%      Value = [term()]
+%%%
+%%% @doc Get a list of all test specification terms used to
+%%% configure and run this test.
+%%%
+get_testspec_terms() ->
+    case ct_util:get_testdata(testspec) of
+	undefined ->
+	    undefined;
+	CurrSpecRec ->
+	    ct_testspec:testspec_rec2list(CurrSpecRec)
+    end.
+
+%%%-----------------------------------------------------------------
+%%% @spec get_testspec_terms(Tags) -> TestSpecTerms | undefined
+%%%      Tags = [Tag] | Tag
+%%%      Tag = atom()
+%%%      TestSpecTerms = [{Tag,Value}] | {Tag,Value}
+%%%      Value = [{Node,term()}] | [term()]
+%%%      Node = atom()
+%%%
+%%% @doc Read one or more terms from the test specification used
+%%% to configure and run this test. Tag is any valid test specification
+%%% tag, such as e.g. <c>label</c>, <c>config</c>, <c>logdir</c>.
+%%% User specific terms are also available to read if the
+%%% <c>allow_user_terms</c> option has been set. Note that all value tuples
+%%% returned, except user terms, will have the node name as first element.
+%%% Note also that in order to read test terms, use <c>Tag = tests</c>
+%%% (rather than <c>suites</c>, <c>groups</c> or <c>cases</c>). Value is
+%%% then the list of *all* tests on the form:
+%%% <c>[{Node,Dir,[{TestSpec,GroupsAndCases1},...]},...], where
+%%% GroupsAndCases = [{Group,[Case]}] | [Case]</c>.
+get_testspec_terms(Tags) ->
+    case ct_util:get_testdata(testspec) of
+	undefined ->
+	    undefined;
+	CurrSpecRec ->
+	    ct_testspec:testspec_rec2list(Tags, CurrSpecRec)
+    end.
+
+
+%%%-----------------------------------------------------------------
 %%% @spec log(Format) -> ok
 %%% @equiv log(default,50,Format,[])
 log(Format) ->
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index ea3d7c8218..6ac16fef4d 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1062,14 +1062,32 @@ get_all_cases1(_, []) ->
 
 get_all(Mod, ConfTests) ->	
     case catch apply(Mod, all, []) of
-	{'EXIT',_} ->
+	{'EXIT',{undef,[{Mod,all,[],_} | _]}} ->
+	    Reason = list_to_atom(atom_to_list(Mod)++":all/0 is missing"),
+	    %% this makes test_server call error_in_suite as first
+	    %% (and only) test case so we can report Reason properly
+	    [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+	{'EXIT',ExitReason} ->
 	    Reason =
 		case code:which(Mod) of
 		    non_existing ->
 			list_to_atom(atom_to_list(Mod)++
-				     " can not be compiled or loaded");
+					 " can not be compiled or loaded");
 		    _ ->
-			list_to_atom(atom_to_list(Mod)++":all/0 is missing")
+			case ct_util:get_testdata({error_in_suite,Mod}) of
+			    undefined ->
+				ErrStr = io_lib:format("~n*** ERROR *** "
+						       "~w:all/0 failed: ~p~n",
+						       [Mod,ExitReason]),
+				io:format(user, ErrStr, []),
+				%% save the error info so it doesn't
+				%% get printed twice
+				ct_util:set_testdata_async({{error_in_suite,Mod},
+							    ExitReason});
+			    _ExitReason ->
+				ct_util:delete_testdata({error_in_suite,Mod})
+			end,
+			list_to_atom(atom_to_list(Mod)++":all/0 failed")
 		end,
 	    %% this makes test_server call error_in_suite as first
 	    %% (and only) test case so we can report Reason properly
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 4d74fd6a80..f6afa423cc 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -77,7 +77,8 @@
 	       multiply_timetraps = 1,
 	       scale_timetraps = false,
 	       create_priv_dir,
-	       testspecs = [],
+	       testspec_files = [],
+	       current_testspec,
 	       tests,
 	       starter}).
 
@@ -485,8 +486,11 @@ execute_one_spec(TS, Opts, Args) ->
     case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
 	ok ->      % read tests from spec
 	    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
-	    do_run(Run, Skip, Opts#opts{config=AllConfig,
-					logdir=TheLogDir}, Args);
+	    Result = do_run(Run, Skip, Opts#opts{config=AllConfig,
+						 logdir=TheLogDir,
+						 current_testspec=TS}, Args),
+	    ct_util:delete_testdata(testspec),
+	    Result;
 	Error ->
 	    Error
     end.
@@ -577,7 +581,7 @@ combine_test_opts(TS, Specs, Opts) ->
 
     Opts#opts{label = Label,
 	      profile = Profile,
-	      testspecs = Specs,
+	      testspec_files = Specs,
 	      cover = Cover,
 	      cover_stop = CoverStop,
 	      logdir = which(logdir, LogDir),
@@ -702,7 +706,7 @@ script_start4(#opts{label = Label, profile = Profile,
 		    logopts = LogOpts,
 		    verbosity = Verbosity,
 		    enable_builtin_hooks = EnableBuiltinHooks,
-		    logdir = LogDir, testspecs = Specs}, _Args) ->
+		    logdir = LogDir, testspec_files = Specs}, _Args) ->
 
     %% label - used by ct_logs
     application:set_env(common_test, test_label, Label),
@@ -1103,7 +1107,7 @@ run_test2(StartOpts) ->
 	undefined ->
 	    case lists:keysearch(prepared_tests, 1, StartOpts) of
 		{value,{_,{Run,Skip},Specs}} ->	% use prepared tests
-		    run_prepared(Run, Skip, Opts#opts{testspecs = Specs},
+		    run_prepared(Run, Skip, Opts#opts{testspec_files = Specs},
 				 StartOpts);
 		false ->
 		    run_dir(Opts, StartOpts)
@@ -1111,11 +1115,11 @@ run_test2(StartOpts) ->
 	Specs ->
 	    Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts),
 	    %% using testspec(s) as input for test
-	    run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts)
+	    run_spec_file(Relaxed, Opts#opts{testspec_files = Specs}, StartOpts)
     end.
 
 run_spec_file(Relaxed,
-	      Opts = #opts{testspecs = Specs},
+	      Opts = #opts{testspec_files = Specs},
 	      StartOpts) ->
     Specs1 = case Specs of
 		 [X|_] when is_integer(X) -> [Specs];
@@ -1399,7 +1403,7 @@ run_testspec2(TestSpec) ->
 	    case check_and_install_configfiles(
 		   Opts#opts.config, LogDir1, Opts) of
 		ok ->
-		    Opts1 = Opts#opts{testspecs = [],
+		    Opts1 = Opts#opts{testspec_files = [],
 				      logdir = LogDir1,
 				      include = AllInclude},
 		    {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
@@ -1706,6 +1710,9 @@ compile_and_run(Tests, Skip, Opts, Args) ->
     ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}),
     %% save logopts
     ct_util:set_testdata({logopts,Opts#opts.logopts}),
+    %% save info about current testspec (testspec record or undefined)
+    ct_util:set_testdata({testspec,Opts#opts.current_testspec}),
+
     %% enable silent connections
     case Opts#opts.silent_connections of
 	[] ->
@@ -1720,7 +1727,7 @@ compile_and_run(Tests, Skip, Opts, Args) ->
 		    ct_logs:log("Silent connections", "~p", [Conns])
 	    end
     end,
-    log_ts_names(Opts#opts.testspecs),
+    log_ts_names(Opts#opts.testspec_files),
     TestSuites = suite_tuples(Tests),
     
     {_TestSuites1,SuiteMakeErrors,AllMakeErrors} =
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 10a9bdac67..10c3f2a938 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -27,6 +27,8 @@
 	 collect_tests_from_list/2, collect_tests_from_list/3,
 	 collect_tests_from_file/2, collect_tests_from_file/3]).
 
+-export([testspec_rec2list/1, testspec_rec2list/2]).
+
 -include("ct_util.hrl").
 -define(testspec_fields, record_info(fields, testspec)).
 
@@ -973,7 +975,8 @@ add_tests([Term={Tag,all_nodes,Data}|Ts],Spec) ->
 					should_be_added(Tag,Node,Data,Spec)],
 	    add_tests(Tests++Ts,Spec);
 	invalid ->				% ignore term
-	    add_tests(Ts,Spec)
+	    Unknown = Spec#testspec.unknown,
+	    add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]})
     end;
 %% create one test entry per node in Nodes and reinsert
 add_tests([{Tag,[],Data}|Ts],Spec) ->
@@ -1001,7 +1004,8 @@ add_tests([Term={Tag,NodeOrOther,Data}|Ts],Spec) ->
 			handle_data(Tag,Node,Data,Spec),
 		    add_tests(Ts,mod_field(Spec,Tag,NodeIxData));
 		invalid ->			% ignore term
-		    add_tests(Ts,Spec)
+		    Unknown = Spec#testspec.unknown,
+		    add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]})
 	    end;
 	false ->
 	    add_tests([{Tag,all_nodes,{NodeOrOther,Data}}|Ts],Spec)
@@ -1012,13 +1016,15 @@ add_tests([Term={Tag,Data}|Ts],Spec) ->
 	valid ->
 	    add_tests([{Tag,all_nodes,Data}|Ts],Spec);
 	invalid ->
-	    add_tests(Ts,Spec)
+	    Unknown = Spec#testspec.unknown,
+	    add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]})
     end;
 %% some other data than a tuple
 add_tests([Other|Ts],Spec) ->	
     case get(relaxed) of
-	true ->		
-	    add_tests(Ts,Spec);
+	true ->
+	    Unknown = Spec#testspec.unknown,
+	    add_tests(Ts,Spec#testspec{unknown=Unknown++[Other]});
 	false ->
 	    throw({error,{undefined_term_in_spec,Other}})
     end;
@@ -1149,6 +1155,24 @@ per_node([N|Ns],Tag,Data,Refs) ->
 per_node([],_,_,_) ->
     [].
 
+%% Change the testspec record "back" to a list of tuples
+testspec_rec2list(Rec) ->
+    {Terms,_} = lists:mapfoldl(fun(unknown, Pos) ->
+				       {element(Pos, Rec),Pos+1};
+				  (F, Pos) ->
+				       {{F,element(Pos, Rec)},Pos+1}
+			       end,2,?testspec_fields),
+    lists:flatten(Terms).
+
+%% Extract one or more values from a testspec record and
+%% return the result as a list of tuples
+testspec_rec2list(Field, Rec) when is_atom(Field) ->
+    [Term] = testspec_rec2list([Field], Rec),
+    Term;
+testspec_rec2list(Fields, Rec) ->
+    Terms = testspec_rec2list(Rec),
+    [{Field,proplists:get_value(Field, Terms)} || Field <- Fields].
+
 %% read the value for FieldName in record Rec#testspec
 read_field(Rec, FieldName) ->
     catch lists:foldl(fun(F, Pos) when F == FieldName ->
diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl
index 845bb55486..f4cf407856 100644
--- a/lib/common_test/src/ct_util.hrl
+++ b/lib/common_test/src/ct_util.hrl
@@ -55,6 +55,7 @@
 		   create_priv_dir=[],
 		   alias=[],
 		   tests=[],
+		   unknown=[],
 		   merge_tests=true}).
 
 -record(cover, {app=none,