From 72d1fe572d3eb3748a86042a818d140f682ebc14 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 5 Jun 2014 16:44:34 +0200 Subject: Introduce test categories for OTP tests --- lib/common_test/doc/src/run_test_chapter.xml | 25 ++++++++++++++++ lib/common_test/src/ct.erl | 45 ++++++++++++++++++++++++++++ lib/common_test/src/ct_framework.erl | 24 +++++++++++++-- lib/common_test/src/ct_run.erl | 27 ++++++++++------- lib/common_test/src/ct_testspec.erl | 34 +++++++++++++++++---- lib/common_test/src/ct_util.hrl | 1 + 6 files changed, 138 insertions(+), 18 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 864f82cb63..df60e5f7f2 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -1005,6 +1005,31 @@ for starting the tests, the relaxed scanner mode is enabled by means of the tuple: {allow_user_terms,true}

+
+ Reading test specification terms +

It's possible to look up terms in the current test specification + (i.e. the spec that's been used to configure and run the current test). + The function get_testspec_terms() returns a list of all test spec + terms (both config- and test terms) and get_testspec_terms(Tags) + returns the term (or a list of terms) matching the tag (or tags) in + Tags.

+

For example, in the test specification:

+ 
+	    ...
+	    {label, my_server_smoke_test}.
+	    {config, "../../my_server_setup.cfg"}.
+	    {config, "../../my_server_interface.cfg"}.
+	    ...
+

And in e.g. a test suite or a CT hook function:

+ 
+	    ...
+	    [{label,[{_Node,TestType}]}, {config,CfgFiles}] =
+                ct:get_testspec_terms([label,config]),
+
+            [verify_my_server_cfg(TestType, CfgFile) || {Node,CfgFile} <- CfgFiles,
+                                                        Node == node()];
+	    ...
+
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 9d8fce2789..5ed1346f1e 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -79,6 +79,7 @@ %% Other interface functions -export([get_status/0, abort_current_testcase/1, get_event_mgr_ref/0, + get_testspec_terms/0, get_testspec_terms/1, encrypt_config_file/2, encrypt_config_file/3, decrypt_config_file/2, decrypt_config_file/3]). @@ -462,6 +463,50 @@ get_config(Required,Default,Opts) -> reload_config(Required)-> ct_config:reload_config(Required). +%%%----------------------------------------------------------------- +%%% @spec get_testspec_terms() -> TestSpecTerms | undefined +%%% TestSpecTerms = [{Tag,Value}] +%%% Value = [term()] +%%% +%%% @doc Get a list of all test specification terms used to +%%% configure and run this test. +%%% +get_testspec_terms() -> + case ct_util:get_testdata(testspec) of + undefined -> + undefined; + CurrSpecRec -> + ct_testspec:testspec_rec2list(CurrSpecRec) + end. + +%%%----------------------------------------------------------------- +%%% @spec get_testspec_terms(Tags) -> TestSpecTerms | undefined +%%% Tags = [Tag] | Tag +%%% Tag = atom() +%%% TestSpecTerms = [{Tag,Value}] | {Tag,Value} +%%% Value = [{Node,term()}] | [term()] +%%% Node = atom() +%%% +%%% @doc Read one or more terms from the test specification used +%%% to configure and run this test. Tag is any valid test specification +%%% tag, such as e.g. label, config, logdir. +%%% User specific terms are also available to read if the +%%% allow_user_terms option has been set. Note that all value tuples +%%% returned, except user terms, will have the node name as first element. +%%% Note also that in order to read test terms, use Tag = tests +%%% (rather than suites, groups or cases). Value is +%%% then the list of *all* tests on the form: +%%% [{Node,Dir,[{TestSpec,GroupsAndCases1},...]},...], where +%%% GroupsAndCases = [{Group,[Case]}] | [Case]. +get_testspec_terms(Tags) -> + case ct_util:get_testdata(testspec) of + undefined -> + undefined; + CurrSpecRec -> + ct_testspec:testspec_rec2list(Tags, CurrSpecRec) + end. + + %%%----------------------------------------------------------------- %%% @spec log(Format) -> ok %%% @equiv log(default,50,Format,[]) diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index ea3d7c8218..6ac16fef4d 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1062,14 +1062,32 @@ get_all_cases1(_, []) -> get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of - {'EXIT',_} -> + {'EXIT',{undef,[{Mod,all,[],_} | _]}} -> + Reason = list_to_atom(atom_to_list(Mod)++":all/0 is missing"), + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Reason properly + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {'EXIT',ExitReason} -> Reason = case code:which(Mod) of non_existing -> list_to_atom(atom_to_list(Mod)++ - " can not be compiled or loaded"); + " can not be compiled or loaded"); _ -> - list_to_atom(atom_to_list(Mod)++":all/0 is missing") + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:all/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(user, ErrStr, []), + %% save the error info so it doesn't + %% get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + list_to_atom(atom_to_list(Mod)++":all/0 failed") end, %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Reason properly diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 4d74fd6a80..f6afa423cc 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -77,7 +77,8 @@ multiply_timetraps = 1, scale_timetraps = false, create_priv_dir, - testspecs = [], + testspec_files = [], + current_testspec, tests, starter}). @@ -485,8 +486,11 @@ execute_one_spec(TS, Opts, Args) -> case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of ok -> % read tests from spec {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig, - logdir=TheLogDir}, Args); + Result = do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir, + current_testspec=TS}, Args), + ct_util:delete_testdata(testspec), + Result; Error -> Error end. @@ -577,7 +581,7 @@ combine_test_opts(TS, Specs, Opts) -> Opts#opts{label = Label, profile = Profile, - testspecs = Specs, + testspec_files = Specs, cover = Cover, cover_stop = CoverStop, logdir = which(logdir, LogDir), @@ -702,7 +706,7 @@ script_start4(#opts{label = Label, profile = Profile, logopts = LogOpts, verbosity = Verbosity, enable_builtin_hooks = EnableBuiltinHooks, - logdir = LogDir, testspecs = Specs}, _Args) -> + logdir = LogDir, testspec_files = Specs}, _Args) -> %% label - used by ct_logs application:set_env(common_test, test_label, Label), @@ -1103,7 +1107,7 @@ run_test2(StartOpts) -> undefined -> case lists:keysearch(prepared_tests, 1, StartOpts) of {value,{_,{Run,Skip},Specs}} -> % use prepared tests - run_prepared(Run, Skip, Opts#opts{testspecs = Specs}, + run_prepared(Run, Skip, Opts#opts{testspec_files = Specs}, StartOpts); false -> run_dir(Opts, StartOpts) @@ -1111,11 +1115,11 @@ run_test2(StartOpts) -> Specs -> Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts), %% using testspec(s) as input for test - run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts) + run_spec_file(Relaxed, Opts#opts{testspec_files = Specs}, StartOpts) end. run_spec_file(Relaxed, - Opts = #opts{testspecs = Specs}, + Opts = #opts{testspec_files = Specs}, StartOpts) -> Specs1 = case Specs of [X|_] when is_integer(X) -> [Specs]; @@ -1399,7 +1403,7 @@ run_testspec2(TestSpec) -> case check_and_install_configfiles( Opts#opts.config, LogDir1, Opts) of ok -> - Opts1 = Opts#opts{testspecs = [], + Opts1 = Opts#opts{testspec_files = [], logdir = LogDir1, include = AllInclude}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), @@ -1706,6 +1710,9 @@ compile_and_run(Tests, Skip, Opts, Args) -> ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), %% save logopts ct_util:set_testdata({logopts,Opts#opts.logopts}), + %% save info about current testspec (testspec record or undefined) + ct_util:set_testdata({testspec,Opts#opts.current_testspec}), + %% enable silent connections case Opts#opts.silent_connections of [] -> @@ -1720,7 +1727,7 @@ compile_and_run(Tests, Skip, Opts, Args) -> ct_logs:log("Silent connections", "~p", [Conns]) end end, - log_ts_names(Opts#opts.testspecs), + log_ts_names(Opts#opts.testspec_files), TestSuites = suite_tuples(Tests), {_TestSuites1,SuiteMakeErrors,AllMakeErrors} = diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 10a9bdac67..10c3f2a938 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -27,6 +27,8 @@ collect_tests_from_list/2, collect_tests_from_list/3, collect_tests_from_file/2, collect_tests_from_file/3]). +-export([testspec_rec2list/1, testspec_rec2list/2]). + -include("ct_util.hrl"). -define(testspec_fields, record_info(fields, testspec)). @@ -973,7 +975,8 @@ add_tests([Term={Tag,all_nodes,Data}|Ts],Spec) -> should_be_added(Tag,Node,Data,Spec)], add_tests(Tests++Ts,Spec); invalid -> % ignore term - add_tests(Ts,Spec) + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]}) end; %% create one test entry per node in Nodes and reinsert add_tests([{Tag,[],Data}|Ts],Spec) -> @@ -1001,7 +1004,8 @@ add_tests([Term={Tag,NodeOrOther,Data}|Ts],Spec) -> handle_data(Tag,Node,Data,Spec), add_tests(Ts,mod_field(Spec,Tag,NodeIxData)); invalid -> % ignore term - add_tests(Ts,Spec) + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]}) end; false -> add_tests([{Tag,all_nodes,{NodeOrOther,Data}}|Ts],Spec) @@ -1012,13 +1016,15 @@ add_tests([Term={Tag,Data}|Ts],Spec) -> valid -> add_tests([{Tag,all_nodes,Data}|Ts],Spec); invalid -> - add_tests(Ts,Spec) + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]}) end; %% some other data than a tuple add_tests([Other|Ts],Spec) -> case get(relaxed) of - true -> - add_tests(Ts,Spec); + true -> + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Other]}); false -> throw({error,{undefined_term_in_spec,Other}}) end; @@ -1149,6 +1155,24 @@ per_node([N|Ns],Tag,Data,Refs) -> per_node([],_,_,_) -> []. +%% Change the testspec record "back" to a list of tuples +testspec_rec2list(Rec) -> + {Terms,_} = lists:mapfoldl(fun(unknown, Pos) -> + {element(Pos, Rec),Pos+1}; + (F, Pos) -> + {{F,element(Pos, Rec)},Pos+1} + end,2,?testspec_fields), + lists:flatten(Terms). + +%% Extract one or more values from a testspec record and +%% return the result as a list of tuples +testspec_rec2list(Field, Rec) when is_atom(Field) -> + [Term] = testspec_rec2list([Field], Rec), + Term; +testspec_rec2list(Fields, Rec) -> + Terms = testspec_rec2list(Rec), + [{Field,proplists:get_value(Field, Terms)} || Field <- Fields]. + %% read the value for FieldName in record Rec#testspec read_field(Rec, FieldName) -> catch lists:foldl(fun(F, Pos) when F == FieldName -> diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 845bb55486..f4cf407856 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -55,6 +55,7 @@ create_priv_dir=[], alias=[], tests=[], + unknown=[], merge_tests=true}). -record(cover, {app=none, -- cgit v1.2.3 From 104f74b2b860f2d82547052ed65acca176ebe34c Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Wed, 15 Apr 2015 00:14:31 +0200 Subject: Add tests for the get_testspec_terms functionality --- lib/common_test/src/ct_run.erl | 5 +++- .../ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl | 28 +++++++++++++++++++++- .../ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl | 2 +- .../ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl | 27 ++++++++++++++++++++- 4 files changed, 58 insertions(+), 4 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index f6afa423cc..33c4f352b0 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1158,7 +1158,10 @@ run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) -> log_ts_names(Specs), Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts), AllConfig = merge_vals([Opts#opts.config, TSConfig]), - try run_one_spec(TS, Combined#opts{config = AllConfig}, StartOpts) of + try run_one_spec(TS, + Combined#opts{config = AllConfig, + current_testspec=TS}, + StartOpts) of Result -> run_all_specs(TSs, Opts, StartOpts, [Result | TotResult]) catch diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl index b8216c3596..cfc6fa93d7 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl @@ -41,8 +41,12 @@ suite() -> %% @end %%-------------------------------------------------------------------- init_per_suite(Config) -> + + TCName = ct:get_config(tcname), + CfgFiles = ct:get_config(file,undefined,[all]), + %% verify that expected config file can be read - case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of + case {TCName,CfgFiles} of {start_separate,[cfg11]} -> ok; {start_join,[cfg11,cfg21]} -> ok; {incl_separate1,[cfg11]} -> ok; @@ -56,6 +60,28 @@ init_per_suite(Config) -> _ -> ok end, + + %% test the get_testspec_terms functionality + if CfgFiles /= undefined -> + TSTerms = case ct:get_testspec_terms() of + undefined -> exit('testspec should not be undefined'); + Result -> Result + end, + true = lists:keymember(config, 1, TSTerms), + {config,TSCfgFiles} = ct:get_testspec_terms(config), + [{config,TSCfgFiles},{tests,Tests}] = + ct:get_testspec_terms([config,tests]), + CfgNames = [list_to_atom(filename:basename(TSCfgFile)) || + {Node,TSCfgFile} <- TSCfgFiles, Node == node()], + true = (length(CfgNames) == length(CfgFiles)), + [true = lists:member(CfgName,CfgFiles) || CfgName <- CfgNames], + true = lists:any(fun({{_Node,_Dir},Suites}) -> + lists:keymember(?MODULE, 1, Suites) + end, Tests); + true -> + ok + end, + Config. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl index 7c51aca246..c3faebbd64 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl @@ -55,7 +55,7 @@ init_per_suite(Config) -> {incl_both2,[cfg11,cfg12,cfg21]} -> ok; {incl_both2,[cfg21]} -> ok; _ -> ok - end, + end, Config. %%-------------------------------------------------------------------- diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl index 36c1b4279b..e189b168c7 100644 --- a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl +++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl @@ -41,8 +41,11 @@ suite() -> %% @end %%-------------------------------------------------------------------- init_per_suite(Config) -> + TCName = ct:get_config(tcname), + CfgFiles = ct:get_config(file,undefined,[all]), + %% verify that expected config file can be read - case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of + case {TCName,CfgFiles} of {start_separate,[cfg11]} -> ok; {start_join,[cfg11,cfg21]} -> ok; {incl_separate1,[cfg11]} -> ok; @@ -55,6 +58,28 @@ init_per_suite(Config) -> {incl_both2,[cfg11]} -> ok; _ -> ok end, + + %% test the get_testspec_terms functionality + if CfgFiles /= undefined -> + TSTerms = case ct:get_testspec_terms() of + undefined -> exit('testspec should not be undefined'); + Result -> Result + end, + true = lists:keymember(config, 1, TSTerms), + {config,TSCfgFiles} = ct:get_testspec_terms(config), + [{config,TSCfgFiles},{tests,Tests}] = + ct:get_testspec_terms([config,tests]), + CfgNames = [list_to_atom(filename:basename(TSCfgFile)) || + {Node,TSCfgFile} <- TSCfgFiles, Node == node()], + true = (length(CfgNames) == length(CfgFiles)), + [true = lists:member(CfgName,CfgFiles) || CfgName <- CfgNames], + true = lists:any(fun({{_Node,_Dir},Suites}) -> + lists:keymember(?MODULE, 1, Suites) + end, Tests); + true -> + ok + end, + Config. %%-------------------------------------------------------------------- -- cgit v1.2.3 From df823580d5030835bbef089d73f833070419d0ff Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Wed, 15 Apr 2015 10:52:29 +0200 Subject: Update handling of failing all/0 function in test suites --- lib/common_test/src/ct_framework.erl | 37 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 6ac16fef4d..a699ec3438 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1063,35 +1063,34 @@ get_all_cases1(_, []) -> get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of {'EXIT',{undef,[{Mod,all,[],_} | _]}} -> - Reason = list_to_atom(atom_to_list(Mod)++":all/0 is missing"), - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Reason properly - [{?MODULE,error_in_suite,[[{error,Reason}]]}]; - {'EXIT',ExitReason} -> Reason = case code:which(Mod) of non_existing -> list_to_atom(atom_to_list(Mod)++ " can not be compiled or loaded"); _ -> - case ct_util:get_testdata({error_in_suite,Mod}) of - undefined -> - ErrStr = io_lib:format("~n*** ERROR *** " - "~w:all/0 failed: ~p~n", - [Mod,ExitReason]), - io:format(user, ErrStr, []), - %% save the error info so it doesn't - %% get printed twice - ct_util:set_testdata_async({{error_in_suite,Mod}, - ExitReason}); - _ExitReason -> - ct_util:delete_testdata({error_in_suite,Mod}) - end, - list_to_atom(atom_to_list(Mod)++":all/0 failed") + list_to_atom(atom_to_list(Mod)++":all/0 is missing") end, %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Reason properly [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {'EXIT',ExitReason} -> + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:all/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(user, ErrStr, []), + %% save the error info so it doesn't get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + Reason = list_to_atom(atom_to_list(Mod)++":all/0 failed"), + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Reason properly + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; AllTCs when is_list(AllTCs) -> case catch save_seqs(Mod,AllTCs) of {error,What} -> -- cgit v1.2.3