diff options
| -rw-r--r-- | lib/common_test/doc/src/run_test_chapter.xml | 25 | ||||
| -rw-r--r-- | lib/common_test/src/ct.erl | 45 | ||||
| -rw-r--r-- | lib/common_test/src/ct_framework.erl | 24 | ||||
| -rw-r--r-- | lib/common_test/src/ct_run.erl | 27 | ||||
| -rw-r--r-- | lib/common_test/src/ct_testspec.erl | 34 | ||||
| -rw-r--r-- | lib/common_test/src/ct_util.hrl | 1 | ||||
| -rw-r--r-- | lib/test_server/src/ts.erl | 557 | ||||
| -rw-r--r-- | lib/test_server/src/ts_lib.erl | 53 |
8 files changed, 509 insertions, 257 deletions
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index 864f82cb63..df60e5f7f2 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -1005,6 +1005,31 @@ for starting the tests, the relaxed scanner mode is enabled by means of the tuple: <c>{allow_user_terms,true}</c></p> </section> + <section> + <title>Reading test specification terms</title> + <p>It's possible to look up terms in the current test specification + (i.e. the spec that's been used to configure and run the current test). + The function <c>get_testspec_terms()</c> returns a list of all test spec + terms (both config- and test terms) and <c>get_testspec_terms(Tags)</c> + returns the term (or a list of terms) matching the tag (or tags) in + <c>Tags</c>.</p> + <p>For example, in the test specification:</p> + <pre> + ... + {label, my_server_smoke_test}. + {config, "../../my_server_setup.cfg"}. + {config, "../../my_server_interface.cfg"}. + ...</pre> + <p>And in e.g. a test suite or a CT hook function:</p> + <pre> + ... + [{label,[{_Node,TestType}]}, {config,CfgFiles}] = + ct:get_testspec_terms([label,config]), + + [verify_my_server_cfg(TestType, CfgFile) || {Node,CfgFile} <- CfgFiles, + Node == node()]; + ...</pre> + </section> </section> <section> diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 9d8fce2789..5ed1346f1e 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -79,6 +79,7 @@ %% Other interface functions -export([get_status/0, abort_current_testcase/1, get_event_mgr_ref/0, + get_testspec_terms/0, get_testspec_terms/1, encrypt_config_file/2, encrypt_config_file/3, decrypt_config_file/2, decrypt_config_file/3]). @@ -463,6 +464,50 @@ reload_config(Required)-> ct_config:reload_config(Required). %%%----------------------------------------------------------------- +%%% @spec get_testspec_terms() -> TestSpecTerms | undefined +%%% TestSpecTerms = [{Tag,Value}] +%%% Value = [term()] +%%% +%%% @doc Get a list of all test specification terms used to +%%% configure and run this test. +%%% +get_testspec_terms() -> + case ct_util:get_testdata(testspec) of + undefined -> + undefined; + CurrSpecRec -> + ct_testspec:testspec_rec2list(CurrSpecRec) + end. + +%%%----------------------------------------------------------------- +%%% @spec get_testspec_terms(Tags) -> TestSpecTerms | undefined +%%% Tags = [Tag] | Tag +%%% Tag = atom() +%%% TestSpecTerms = [{Tag,Value}] | {Tag,Value} +%%% Value = [{Node,term()}] | [term()] +%%% Node = atom() +%%% +%%% @doc Read one or more terms from the test specification used +%%% to configure and run this test. Tag is any valid test specification +%%% tag, such as e.g. <c>label</c>, <c>config</c>, <c>logdir</c>. +%%% User specific terms are also available to read if the +%%% <c>allow_user_terms</c> option has been set. Note that all value tuples +%%% returned, except user terms, will have the node name as first element. +%%% Note also that in order to read test terms, use <c>Tag = tests</c> +%%% (rather than <c>suites</c>, <c>groups</c> or <c>cases</c>). Value is +%%% then the list of *all* tests on the form: +%%% <c>[{Node,Dir,[{TestSpec,GroupsAndCases1},...]},...], where +%%% GroupsAndCases = [{Group,[Case]}] | [Case]</c>. +get_testspec_terms(Tags) -> + case ct_util:get_testdata(testspec) of + undefined -> + undefined; + CurrSpecRec -> + ct_testspec:testspec_rec2list(Tags, CurrSpecRec) + end. + + +%%%----------------------------------------------------------------- %%% @spec log(Format) -> ok %%% @equiv log(default,50,Format,[]) log(Format) -> diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index ea3d7c8218..6ac16fef4d 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1062,14 +1062,32 @@ get_all_cases1(_, []) -> get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of - {'EXIT',_} -> + {'EXIT',{undef,[{Mod,all,[],_} | _]}} -> + Reason = list_to_atom(atom_to_list(Mod)++":all/0 is missing"), + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Reason properly + [{?MODULE,error_in_suite,[[{error,Reason}]]}]; + {'EXIT',ExitReason} -> Reason = case code:which(Mod) of non_existing -> list_to_atom(atom_to_list(Mod)++ - " can not be compiled or loaded"); + " can not be compiled or loaded"); _ -> - list_to_atom(atom_to_list(Mod)++":all/0 is missing") + case ct_util:get_testdata({error_in_suite,Mod}) of + undefined -> + ErrStr = io_lib:format("~n*** ERROR *** " + "~w:all/0 failed: ~p~n", + [Mod,ExitReason]), + io:format(user, ErrStr, []), + %% save the error info so it doesn't + %% get printed twice + ct_util:set_testdata_async({{error_in_suite,Mod}, + ExitReason}); + _ExitReason -> + ct_util:delete_testdata({error_in_suite,Mod}) + end, + list_to_atom(atom_to_list(Mod)++":all/0 failed") end, %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Reason properly diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 4d74fd6a80..f6afa423cc 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -77,7 +77,8 @@ multiply_timetraps = 1, scale_timetraps = false, create_priv_dir, - testspecs = [], + testspec_files = [], + current_testspec, tests, starter}). @@ -485,8 +486,11 @@ execute_one_spec(TS, Opts, Args) -> case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of ok -> % read tests from spec {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig, - logdir=TheLogDir}, Args); + Result = do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir, + current_testspec=TS}, Args), + ct_util:delete_testdata(testspec), + Result; Error -> Error end. @@ -577,7 +581,7 @@ combine_test_opts(TS, Specs, Opts) -> Opts#opts{label = Label, profile = Profile, - testspecs = Specs, + testspec_files = Specs, cover = Cover, cover_stop = CoverStop, logdir = which(logdir, LogDir), @@ -702,7 +706,7 @@ script_start4(#opts{label = Label, profile = Profile, logopts = LogOpts, verbosity = Verbosity, enable_builtin_hooks = EnableBuiltinHooks, - logdir = LogDir, testspecs = Specs}, _Args) -> + logdir = LogDir, testspec_files = Specs}, _Args) -> %% label - used by ct_logs application:set_env(common_test, test_label, Label), @@ -1103,7 +1107,7 @@ run_test2(StartOpts) -> undefined -> case lists:keysearch(prepared_tests, 1, StartOpts) of {value,{_,{Run,Skip},Specs}} -> % use prepared tests - run_prepared(Run, Skip, Opts#opts{testspecs = Specs}, + run_prepared(Run, Skip, Opts#opts{testspec_files = Specs}, StartOpts); false -> run_dir(Opts, StartOpts) @@ -1111,11 +1115,11 @@ run_test2(StartOpts) -> Specs -> Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts), %% using testspec(s) as input for test - run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts) + run_spec_file(Relaxed, Opts#opts{testspec_files = Specs}, StartOpts) end. run_spec_file(Relaxed, - Opts = #opts{testspecs = Specs}, + Opts = #opts{testspec_files = Specs}, StartOpts) -> Specs1 = case Specs of [X|_] when is_integer(X) -> [Specs]; @@ -1399,7 +1403,7 @@ run_testspec2(TestSpec) -> case check_and_install_configfiles( Opts#opts.config, LogDir1, Opts) of ok -> - Opts1 = Opts#opts{testspecs = [], + Opts1 = Opts#opts{testspec_files = [], logdir = LogDir1, include = AllInclude}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), @@ -1706,6 +1710,9 @@ compile_and_run(Tests, Skip, Opts, Args) -> ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}), %% save logopts ct_util:set_testdata({logopts,Opts#opts.logopts}), + %% save info about current testspec (testspec record or undefined) + ct_util:set_testdata({testspec,Opts#opts.current_testspec}), + %% enable silent connections case Opts#opts.silent_connections of [] -> @@ -1720,7 +1727,7 @@ compile_and_run(Tests, Skip, Opts, Args) -> ct_logs:log("Silent connections", "~p", [Conns]) end end, - log_ts_names(Opts#opts.testspecs), + log_ts_names(Opts#opts.testspec_files), TestSuites = suite_tuples(Tests), {_TestSuites1,SuiteMakeErrors,AllMakeErrors} = diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 10a9bdac67..10c3f2a938 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -27,6 +27,8 @@ collect_tests_from_list/2, collect_tests_from_list/3, collect_tests_from_file/2, collect_tests_from_file/3]). +-export([testspec_rec2list/1, testspec_rec2list/2]). + -include("ct_util.hrl"). -define(testspec_fields, record_info(fields, testspec)). @@ -973,7 +975,8 @@ add_tests([Term={Tag,all_nodes,Data}|Ts],Spec) -> should_be_added(Tag,Node,Data,Spec)], add_tests(Tests++Ts,Spec); invalid -> % ignore term - add_tests(Ts,Spec) + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]}) end; %% create one test entry per node in Nodes and reinsert add_tests([{Tag,[],Data}|Ts],Spec) -> @@ -1001,7 +1004,8 @@ add_tests([Term={Tag,NodeOrOther,Data}|Ts],Spec) -> handle_data(Tag,Node,Data,Spec), add_tests(Ts,mod_field(Spec,Tag,NodeIxData)); invalid -> % ignore term - add_tests(Ts,Spec) + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]}) end; false -> add_tests([{Tag,all_nodes,{NodeOrOther,Data}}|Ts],Spec) @@ -1012,13 +1016,15 @@ add_tests([Term={Tag,Data}|Ts],Spec) -> valid -> add_tests([{Tag,all_nodes,Data}|Ts],Spec); invalid -> - add_tests(Ts,Spec) + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]}) end; %% some other data than a tuple add_tests([Other|Ts],Spec) -> case get(relaxed) of - true -> - add_tests(Ts,Spec); + true -> + Unknown = Spec#testspec.unknown, + add_tests(Ts,Spec#testspec{unknown=Unknown++[Other]}); false -> throw({error,{undefined_term_in_spec,Other}}) end; @@ -1149,6 +1155,24 @@ per_node([N|Ns],Tag,Data,Refs) -> per_node([],_,_,_) -> []. +%% Change the testspec record "back" to a list of tuples +testspec_rec2list(Rec) -> + {Terms,_} = lists:mapfoldl(fun(unknown, Pos) -> + {element(Pos, Rec),Pos+1}; + (F, Pos) -> + {{F,element(Pos, Rec)},Pos+1} + end,2,?testspec_fields), + lists:flatten(Terms). + +%% Extract one or more values from a testspec record and +%% return the result as a list of tuples +testspec_rec2list(Field, Rec) when is_atom(Field) -> + [Term] = testspec_rec2list([Field], Rec), + Term; +testspec_rec2list(Fields, Rec) -> + Terms = testspec_rec2list(Rec), + [{Field,proplists:get_value(Field, Terms)} || Field <- Fields]. + %% read the value for FieldName in record Rec#testspec read_field(Rec, FieldName) -> catch lists:foldl(fun(F, Pos) when F == FieldName -> diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 845bb55486..f4cf407856 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -55,6 +55,7 @@ create_priv_dir=[], alias=[], tests=[], + unknown=[], merge_tests=true}). -record(cover, {app=none, diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index d6d2e865e2..c2a7afa38b 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -24,11 +24,11 @@ -module(ts). --export([run/0, run/1, run/2, run/3, run/4, run/5, - tests/0, tests/1, +-export([cl_run/1, + run/0, run/1, run/2, run/3, run/4, run/5, + run_category/1, run_category/2, run_category/3, + tests/0, tests/1, suites/1, categories/1, install/0, install/1, - bench/0, bench/1, bench/2, benchmarks/0, - smoke_test/0, smoke_test/1,smoke_test/2, smoke_tests/0, estone/0, estone/1, cross_cover_analyse/1, compile_testcases/0, compile_testcases/1, @@ -82,10 +82,13 @@ -define( install_help, [ - " ts:install() - Install TS with no Options.\n" - " ts:install([Options]) - Install TS with Options\n" + " ts:install()\n", + " Install ts with no options.\n", "\n", - "Installation options supported:\n", + " ts:install(Options)\n", + " Install ts with a list of options, see below.\n", + "\n", + "Installation options supported:\n\n", " {longnames, true} - Use fully qualified hostnames\n", " {verbose, Level} - Sets verbosity level for TS output (0,1,2), 0 is\n" " quiet(default).\n" @@ -110,21 +113,64 @@ help() -> end. help(uninstalled) -> - H = ["TS is not installed yet. To install use:\n\n"], + H = ["ts is not yet installed. To install use:\n\n"], show_help([H,?install_help]); help(installed) -> - H = ["Run functions:\n", - " ts:run() - Run all available tests.\n", - " ts:run(Spec) - Run all tests in given test spec file.\n", - " The spec file is actually ../*_test/Spec.spec\n", - " ts:run([Specs]) - Run all tests in all given test spec files.\n", - " ts:run(Spec, Mod) - Run a single test suite.\n", - " ts:run(Spec, Mod, Case)\n", - " - Run a single test case.\n", - " All above run functions can have an additional Options argument\n", - " which is a list of options.\n", + H = ["\n", + "Run functions:\n\n", + " ts:run()\n", + " Run the tests for all apps. The tests are defined by the\n", + " main test specification for each app: ../App_test/App.spec.\n", + "\n", + " ts:run(Apps)\n", + " Apps = atom() | [atom()]\n", + " Run the tests for an app, or set of apps. The tests are\n", + " defined by the main test specification for each app:\n", + " ../App_test/App.spec.\n", + "\n", + " ts:run(App, Suites)\n", + " App = atom(), Suites = atom() | [atom()]\n", + " Run one or more test suites for App (i.e. modules named\n", + " *_SUITE.erl, located in ../App_test/).\n", + "\n", + " ts:run(App, Suite, TestCases)\n", + " App = atom(), Suite = atom(),\n", + " TestCases = TCs | {testcase,TCs}, TCs = atom() | [atom()]\n", + " Run one or more test cases (functions) in Suite.\n", + "\n", + " ts:run(App, Suite, {group,Groups})\n", + " App = atom(), Suite = atom(), Groups = atom() | [atom()]\n", + " Run one or more test case groups in Suite.\n", + "\n", + " ts:run(App, Suite, {group,Group}, {testcase,TestCases})\n", + " App = atom(), Suite = atom(), Group = atom(),\n", + " TestCases = atom() | [atom()]\n", + " Run one or more test cases in a test case group in Suite.\n", + "\n", + " ts:run_category(TestCategory)\n", + " TestCategory = smoke | essential | bench | atom()\n", + " Run the specified category of tests for all apps.\n", + " For each app, the tests are defined by the specification:\n", + " ../App_test/App_TestCategory.spec.\n", + "\n", + " ts:run_category(Apps, TestCategory)\n", + " Apps = atom() | [atom()],\n", + " TestCategory = smoke | essential | bench | atom()\n", + " Run the specified category of tests for the given app or apps.\n", "\n", - "Run options supported:\n", + " Note that the test category parameter may have arbitrary value,\n", + " but should correspond to an existing test specification with file\n", + " name: ../App_test/App_TestCategory.spec.\n", + " Predefined categories exist for smoke tests, essential tests and\n", + " benchmark tests. The corresponding specs are:\n", + " ../*_test/Spec_smoke.spec, ../*_test/Spec_essential.spec and\n", + " ../*_test/Spec_bench.spec.\n", + "\n", + " All above run functions can take an additional last argument,\n", + " Options, which is a list of options (e.g. ts:run(App, Options),\n", + " or ts:run_category(Apps, TestCategory, Options)).\n", + "\n", + "Run options supported:\n\n", " batch - Do not start a new xterm\n", " {verbose, Level} - Same as the verbosity option for install\n", " verbose - Same as {verbose, 1}\n", @@ -143,47 +189,46 @@ help(installed) -> " files are. The default location is\n" " tests/test_server/.\n" "\n", - "Supported trace information elements\n", + "Supported trace information elements:\n\n", " {tp | tpl, Mod, [] | match_spec()}\n", " {tp | tpl, Mod, Func, [] | match_spec()}\n", " {tp | tpl, Mod, Func, Arity, [] | match_spec()}\n", " {ctp | ctpl, Mod}\n", " {ctp | ctpl, Mod, Func}\n", " {ctp | ctpl, Mod, Func, Arity}\n", + "\n\n", + "Support functions:\n\n", + " ts:tests()\n", + " Returns all apps available for testing.\n", + "\n", + " ts:tests(TestCategory)\n", + " Returns all apps that provide tests in the given category.\n", + "\n", + " ts:suites(App)\n", + " Returns all available test suites for App,\n", + " i.e. ../App_test/*_SUITE.erl\n", + "\n", + " ts:categories(App)\n", + " Returns all test categories available for App.\n", + "\n", + " ts:estone()\n", + " Runs estone_SUITE in the kernel application with no run options\n", + "\n", + " ts:estone(Opts)\n", + " Runs estone_SUITE in the kernel application with the given\n", + " run options\n", + "\n", + " ts:cross_cover_analyse(Level)\n", + " Use after ts:run with option cover or cover_details. Analyses\n", + " modules specified with a 'cross' statement in the cover spec file.\n", + " Level can be 'overview' or 'details'.\n", "\n", - "Support functions:\n", - " ts:tests() - Shows all available families of tests.\n", - " ts:tests(Spec) - Shows all available test modules in Spec,\n", - " i.e. ../Spec_test/*_SUITE.erl\n", - " ts:estone() - Run estone_SUITE in kernel application with\n" - " no run options\n", - " ts:estone(Opts) - Run estone_SUITE in kernel application with\n" - " the given run options\n", - " ts:cross_cover_analyse(Level)\n" - " - Used after ts:run with option cover or \n" - " cover_details. Analyses modules specified with\n" - " a 'cross' statement in the cover spec file.\n" - " Level can be 'overview' or 'details'.\n", - " ts:compile_testcases()~n" - " ts:compile_testcases(Apps)~n" - " - Compile all testcases for usage in a cross ~n" - " compile environment." - " \n" - "Benchmark functions:\n" - " ts:benchmarks() - Get all available families of benchmarks\n" - " ts:bench() - Runs all benchmarks\n" - " ts:bench(Spec) - Runs all benchmarks in the given spec file.\n" - " The spec file is actually ../*_test/Spec_bench.spec\n\n" - " ts:bench can take the same Options argument as ts:run.\n" - "Smoke test functions:\n" - " ts:smoke_tests() - Get all available families of smoke tests\n" - " ts:smoke_test() - Runs all smoke tests\n" - " ts:smoke_test(Spec)\n" - " - Runs all smoke tests in the given spec file.\n" - " The spec file is actually ../*_test/Spec_smoke.spec\n\n" - " ts:smoke_test can take the same Options argument as ts:run.\n" - "\n" - "Installation (already done):\n" + " ts:compile_testcases()\n", + " ts:compile_testcases(Apps)\n", + " Compiles all test cases for the given apps, for usage in a\n", + " cross compilation environment.\n", + "\n\n", + "Installation (already done):\n\n" ], show_help([H,?install_help]). @@ -212,86 +257,128 @@ run_all(_Vars) -> run_some([], _Opts) -> ok; -run_some([{Spec,Mod}|Specs], Opts) -> - case run(Spec, Mod, Opts) of +run_some([{App,Mod}|Apps], Opts) -> + case run(App, Mod, Opts) of ok -> ok; - Error -> io:format("~p: ~p~n",[{Spec,Mod},Error]) + Error -> io:format("~p: ~p~n",[{App,Mod},Error]) end, - run_some(Specs, Opts); -run_some([Spec|Specs], Opts) -> - case run(Spec, Opts) of + run_some(Apps, Opts); +run_some([App|Apps], Opts) -> + case run(App, Opts) of ok -> ok; - Error -> io:format("~p: ~p~n",[Spec,Error]) + Error -> io:format("~p: ~p~n",[App,Error]) end, - run_some(Specs, Opts). + run_some(Apps, Opts). + +%% This can be used from command line. Both App and +%% TestCategory must be specified. App may be 'all' +%% and TestCategory may be 'main'. Examples: +%% erl -s ts cl_run kernel smoke <options> +%% erl -s ts cl_run kernel main <options> +%% erl -s ts cl_run all essential <options> +%% erl -s ts cl_run all main <options> +%% When using the 'main' category and running with cover, +%% one can also use the cross_cover_analysis flag. +cl_run([App,Cat|Options0]) when is_atom(App) -> -%% Runs one test spec (interactive). -run(Testspec) when is_atom(Testspec) -> - Options=check_test_get_opts(Testspec, []), - File = atom_to_list(Testspec), - run_test(File, [{spec,[File++".spec"]}], Options); - -%% This can be used from command line, e.g. -%% erl -s ts run all_tests <config> -%% When using the all_tests flag and running with cover, one can also -%% use the cross_cover_analysis flag. -run([all_tests|Config0]) -> AllAtomsFun = fun(X) when is_atom(X) -> true; (_) -> false end, - Config1 = - case lists:all(AllAtomsFun,Config0) of + Options1 = + case lists:all(AllAtomsFun, Options0) of true -> %% Could be from command line - lists:map(fun(Conf)->to_erlang_term(Conf) end,Config0)--[batch]; + lists:map(fun(Opt) -> + to_erlang_term(Opt) + end, Options0) -- [batch]; false -> - Config0--[batch] + Options0 -- [batch] end, %% Make sure there is exactly one occurence of 'batch' - Config2 = [batch|Config1], - - R = run(tests(),Config2), - - case check_for_cross_cover_analysis_flag(Config2) of + Options2 = [batch|Options1], + + Result = + case {App,Cat} of + {all,main} -> + run(tests(), Options2); + {all,Cat} -> + run_category(Cat, Options2); + {_,main} -> + run(App, Options2); + {_,Cat} -> + run_category(App, Cat, Options2) + end, + case check_for_cross_cover_analysis_flag(Options2) of false -> ok; Level -> cross_cover_analyse(Level) end, + Result. - R; - -%% ts:run(ListOfTests) -run(List) when is_list(List) -> - run(List, [batch]). +%% run/1 +%% Runs tests for one app (interactive). +run(App) when is_atom(App) -> + Options = check_test_get_opts(App, []), + File = atom_to_list(App), + run_test(File, [{spec,[File++".spec"]},{allow_user_terms,true}], Options); -run(List, Opts) when is_list(List), is_list(Opts) -> - run_some(List, Opts); +%% This can be used from command line, e.g. +%% erl -s ts run all <options> +%% erl -s ts run main <options> +run([all,main|Opts]) -> + cl_run([all,main|Opts]); +run([all|Opts]) -> + cl_run([all,main|Opts]); +run([main|Opts]) -> + cl_run([all,main|Opts]); +%% Backwards compatible +run([all_tests|Opts]) -> + cl_run([all,main|Opts]); + +%% run/1 +%% Runs the main tests for all available apps +run(Apps) when is_list(Apps) -> + run(Apps, [batch]). %% run/2 -%% Runs one test spec with list of suites or with options -run(Testspec, ModsOrConfig) when is_atom(Testspec), - is_list(ModsOrConfig) -> - case is_list_of_suites(ModsOrConfig) of +%% Runs the main tests for all available apps +run(Apps, Opts) when is_list(Apps), is_list(Opts) -> + run_some(Apps, Opts); + +%% Runs tests for one app with list of suites or with options +run(App, ModsOrOpts) when is_atom(App), + is_list(ModsOrOpts) -> + case is_list_of_suites(ModsOrOpts) of false -> - run(Testspec, {config_list,ModsOrConfig}); + run(App, {opts_list,ModsOrOpts}); true -> - run_some([{Testspec,M} || M <- ModsOrConfig], + run_some([{App,M} || M <- ModsOrOpts], [batch]) end; -run(Testspec, {config_list,Config}) -> - Options=check_test_get_opts(Testspec, Config), - IsSmoke=proplists:get_value(smoke,Config), - File=atom_to_list(Testspec), + +run(App, {opts_list,Opts}) -> + Options = check_test_get_opts(App, Opts), + File = atom_to_list(App), + + %% check if other test category than main has been specified + {CatSpecName,TestCat} = + case proplists:get_value(test_category, Opts) of + undefined -> + {"",main}; + Cat -> + {"_" ++ atom_to_list(Cat),Cat} + end, + WhatToDo = - case Testspec of + case App of %% Known to exist but fails generic tests below emulator -> test; system -> test; erl_interface -> test; epmd -> test; _ -> - case code:lib_dir(Testspec) of + case code:lib_dir(App) of {error,bad_name} -> %% Application does not exist skip; @@ -313,92 +400,135 @@ run(Testspec, {config_list,Config}) -> end end end, - Spec = - case WhatToDo of - skip -> - create_skip_spec(Testspec, tests(Testspec)); - test when IsSmoke -> - File++"_smoke.spec"; - test -> - File++".spec" - end, - run_test(File, [{spec,[Spec]}], Options); -%% Runs one module in a spec (interactive) -run(Testspec, Mod) when is_atom(Testspec), is_atom(Mod) -> - run_test({atom_to_list(Testspec),Mod}, + case WhatToDo of + skip -> + SkipSpec = create_skip_spec(App, suites(App)), + run_test(File, [{spec,[SkipSpec]}], Options); + test when TestCat == bench -> + check_and_run(fun(Vars) -> + ts_benchmark:run([App], Options, Vars) + end); + test -> + Spec = File ++ CatSpecName ++ ".spec", + run_test(File, [{spec,[Spec]},{allow_user_terms,true}], Options) + end; + +%% Runs one module for an app (interactive) +run(App, Mod) when is_atom(App), is_atom(Mod) -> + run_test({atom_to_list(App),Mod}, [{suite,Mod}], [interactive]). %% run/3 -%% Run one module in a spec with Config -run(Testspec, Mod, Config) when is_atom(Testspec), - is_atom(Mod), - is_list(Config) -> - Options=check_test_get_opts(Testspec, Config), - run_test({atom_to_list(Testspec),Mod}, +%% Run one module for an app with Opts +run(App, Mod, Opts) when is_atom(App), + is_atom(Mod), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), + run_test({atom_to_list(App),Mod}, [{suite,Mod}], Options); -%% Run multiple modules with Config -run(Testspec, Mods, Config) when is_atom(Testspec), - is_list(Mods), - is_list(Config) -> - run_some([{Testspec,M} || M <- Mods], Config); + +%% Run multiple modules with Opts +run(App, Mods, Opts) when is_atom(App), + is_list(Mods), + is_list(Opts) -> + run_some([{App,M} || M <- Mods], Opts); + %% Runs one test case in a module. -run(Testspec, Mod, Case) when is_atom(Testspec), - is_atom(Mod), - is_atom(Case) -> - Options=check_test_get_opts(Testspec, []), +run(App, Mod, Case) when is_atom(App), + is_atom(Mod), + is_atom(Case) -> + Options = check_test_get_opts(App, []), Args = [{suite,Mod},{testcase,Case}], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + %% Runs one or more groups in a module. -run(Testspec, Mod, Grs={group,_Groups}) when is_atom(Testspec), - is_atom(Mod) -> - Options=check_test_get_opts(Testspec, []), +run(App, Mod, Grs={group,_Groups}) when is_atom(App), + is_atom(Mod) -> + Options = check_test_get_opts(App, []), Args = [{suite,Mod},Grs], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + %% Runs one or more test cases in a module. -run(Testspec, Mod, TCs={testcase,_Cases}) when is_atom(Testspec), - is_atom(Mod) -> - Options=check_test_get_opts(Testspec, []), +run(App, Mod, TCs={testcase,_Cases}) when is_atom(App), + is_atom(Mod) -> + Options = check_test_get_opts(App, []), Args = [{suite,Mod},TCs], - run_test(atom_to_list(Testspec), Args, Options). + run_test(atom_to_list(App), Args, Options). %% run/4 %% Run one test case in a module with Options. -run(Testspec, Mod, Case, Config) when is_atom(Testspec), - is_atom(Mod), - is_atom(Case), - is_list(Config) -> - Options=check_test_get_opts(Testspec, Config), +run(App, Mod, Case, Opts) when is_atom(App), + is_atom(Mod), + is_atom(Case), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},{testcase,Case}], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + %% Run one or more test cases in a module with Options. -run(Testspec, Mod, {testcase,Cases}, Config) when is_atom(Testspec), - is_atom(Mod) -> - run(Testspec, Mod, Cases, Config); -run(Testspec, Mod, Cases, Config) when is_atom(Testspec), - is_atom(Mod), - is_list(Cases), - is_list(Config) -> - Options=check_test_get_opts(Testspec, Config), +run(App, Mod, {testcase,Cases}, Opts) when is_atom(App), + is_atom(Mod) -> + run(App, Mod, Cases, Opts); +run(App, Mod, Cases, Opts) when is_atom(App), + is_atom(Mod), + is_list(Cases), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},Cases], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + +%% Run one or more test cases in a group. +run(App, Mod, Gr={group,_Group}, {testcase,Cases}) when is_atom(App), + is_atom(Mod) -> + run(App, Mod, Gr, Cases, [batch]); + + %% Run one or more groups in a module with Options. -run(Testspec, Mod, Grs={group,_Groups}, Config) when is_atom(Testspec), - is_atom(Mod) -> - Options=check_test_get_opts(Testspec, Config), +run(App, Mod, Grs={group,_Groups}, Opts) when is_atom(App), + is_atom(Mod), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},Grs], - run_test(atom_to_list(Testspec), Args, Options). + run_test(atom_to_list(App), Args, Options). %% run/5 %% Run one or more test cases in a group with Options. -run(Testspec, Mod, Group, Cases, Config) when is_atom(Testspec), - is_atom(Mod), - is_list(Config) -> +run(App, Mod, Group, Cases, Opts) when is_atom(App), + is_atom(Mod), + is_list(Opts) -> Group1 = if is_tuple(Group) -> Group; true -> {group,Group} end, Cases1 = if is_tuple(Cases) -> Cases; true -> {testcase,Cases} end, - Options=check_test_get_opts(Testspec, Config), + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},Group1,Cases1], - run_test(atom_to_list(Testspec), Args, Options). + run_test(atom_to_list(App), Args, Options). + +%% run_category/1 +run_category(TestCategory) when is_atom(TestCategory) -> + run_category(TestCategory, [batch]). + +%% run_category/2 +run_category(TestCategory, Opts) when is_atom(TestCategory), + is_list(Opts) -> + case ts:tests() of + [] -> + {error, no_tests_available}; + Apps -> + Opts1 = [{test_category,TestCategory} | Opts], + run_some(Apps, Opts1) + end; + +run_category(Apps, TestCategory) when is_atom(TestCategory) -> + run_category(Apps, TestCategory, [batch]). + +%% run_category/3 +run_category(App, TestCategory, Opts) -> + Apps = if is_atom(App) -> [App]; + is_list(App) -> App + end, + Opts1 = [{test_category,TestCategory} | Opts], + run_some(Apps, Opts1). + is_list_of_suites(List) -> lists:all(fun(Suite) -> @@ -416,29 +546,29 @@ is_list_of_suites(List) -> %% Create a spec to skip all SUITES, this is used when the application %% to be tested is not part of the OTP release to be tested. -create_skip_spec(Testspec, SuitesToSkip) -> +create_skip_spec(App, SuitesToSkip) -> {ok,Cwd} = file:get_cwd(), - TestspecString = atom_to_list(Testspec), - Specname = TestspecString++"_skip.spec", + AppString = atom_to_list(App), + Specname = AppString++"_skip.spec", {ok,D} = file:open(filename:join([filename:dirname(Cwd), - TestspecString++"_test",Specname]), + AppString++"_test",Specname]), [write]), - TestDir = "\"../"++TestspecString++"_test\"", + TestDir = "\"../"++AppString++"_test\"", io:format(D,"{suites, "++TestDir++", all}.~n",[]), io:format(D,"{skip_suites, "++TestDir++", ~w, \"Skipped as application" " is not in path!\"}.",[SuitesToSkip]), Specname. -%% Check testspec to be valid and get possible Options -%% from the config. -check_test_get_opts(Testspec, Config) -> - validate_test(Testspec), - Mode = configmember(batch, {batch, interactive}, Config), - Vars = configvars(Config), - Trace = get_config(trace,Config), - ConfigPath = get_config(config,Config), - KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Config), - Cover = configcover(Testspec,Config), +%% Check testspec for App to be valid and get possible options +%% from the list. +check_test_get_opts(App, Opts) -> + validate_test(App), + Mode = configmember(batch, {batch, interactive}, Opts), + Vars = configvars(Opts), + Trace = get_config(trace,Opts), + ConfigPath = get_config(config,Opts), + KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Opts), + Cover = configcover(App,Opts), lists:flatten([Vars,Mode,Trace,KeepTopcase,Cover,ConfigPath]). to_erlang_term(Atom) -> @@ -447,7 +577,7 @@ to_erlang_term(Atom) -> {ok, Term} = erl_parse:parse_term(Tokens), Term. -%% Validate that a Testspec really is a testspec, +%% Validate that Testspec really is a testspec, %% and exit if not. validate_test(Testspec) -> case lists:member(Testspec, tests()) of @@ -460,10 +590,10 @@ validate_test(Testspec) -> exit(self(), {error, test_not_available}) end. -configvars(Config) -> - case lists:keysearch(vars, 1, Config) of +configvars(Opts) -> + case lists:keysearch(vars, 1, Opts) of {value, {vars, List}} -> - List0 = special_vars(Config), + List0 = special_vars(Opts), Key = fun(T) -> element(1,T) end, DelDupList = lists:filter(fun(V) -> @@ -474,17 +604,17 @@ configvars(Config) -> end, List), {vars, [List0|DelDupList]}; _ -> - {vars, special_vars(Config)} + {vars, special_vars(Opts)} end. -%% Allow some shortcuts in the Options... -special_vars(Config) -> +%% Allow some shortcuts in the options... +special_vars(Opts) -> SpecVars = - case lists:member(verbose, Config) of + case lists:member(verbose, Opts) of true -> [{verbose, 1}]; false -> - case lists:keysearch(verbose, 1, Config) of + case lists:keysearch(verbose, 1, Opts) of {value, {verbose, Lvl}} -> [{verbose, Lvl}]; _ -> @@ -492,13 +622,13 @@ special_vars(Config) -> end end, SpecVars1 = - case lists:keysearch(diskless, 1, Config) of + case lists:keysearch(diskless, 1, Opts) of {value,{diskless, true}} -> [{diskless, true} | SpecVars]; _ -> SpecVars end, - case lists:keysearch(testcase_callback, 1, Config) of + case lists:keysearch(testcase_callback, 1, Opts) of {value,{testcase_callback, CBM, CBF}} -> [{ts_testcase_callback, {CBM,CBF}} | SpecVars1]; {value,{testcase_callback, CB}} -> @@ -566,50 +696,31 @@ check_for_cross_cover_analysis_flag([_|Config],Level,CrossFlag) -> check_for_cross_cover_analysis_flag([],_,_) -> false. -%% Returns a list of available test suites. +%% Returns all available apps. tests() -> {ok, Cwd} = file:get_cwd(), ts_lib:specs(Cwd). -tests(Spec) -> +%% Returns all apps that provide tests in the given test category +tests(main) -> {ok, Cwd} = file:get_cwd(), - ts_lib:suites(Cwd, atom_to_list(Spec)). - -%% Benchmark related functions - -bench() -> - bench([]). - -bench(Opts) when is_list(Opts) -> - bench(benchmarks(),Opts); -bench(Spec) -> - bench([Spec],[]). - -bench(Spec, Opts) when is_atom(Spec) -> - bench([Spec],Opts); -bench(Specs, Opts) -> - check_and_run(fun(Vars) -> ts_benchmark:run(Specs, Opts, Vars) end). - -benchmarks() -> - ts_benchmark:benchmarks(). - -smoke_test() -> - smoke_test([]). - -smoke_test(Opts) when is_list(Opts) -> - smoke_test(smoke_tests(),Opts); -smoke_test(Spec) -> - smoke_test([Spec],[]). - -smoke_test(Spec, Opts) when is_atom(Spec) -> - smoke_test([Spec],Opts); -smoke_test(Specs, Opts) -> - run(Specs, [{smoke,true}|Opts]). + ts_lib:specs(Cwd); +tests(bench) -> + ts_benchmark:benchmarks(); +tests(TestCategory) -> + {ok, Cwd} = file:get_cwd(), + ts_lib:specialized_specs(Cwd, atom_to_list(TestCategory)). + +%% Returns a list of available test suites for App. +suites(App) -> + {ok, Cwd} = file:get_cwd(), + ts_lib:suites(Cwd, atom_to_list(App)). -smoke_tests() -> +%% Returns all available test categories for App +categories(App) -> {ok, Cwd} = file:get_cwd(), - ts_lib:specialized_specs(Cwd,"smoke"). + ts_lib:test_categories(Cwd, atom_to_list(App)). %% %% estone/0, estone/1 diff --git a/lib/test_server/src/ts_lib.erl b/lib/test_server/src/ts_lib.erl index 5368960446..45746c444d 100644 --- a/lib/test_server/src/ts_lib.erl +++ b/lib/test_server/src/ts_lib.erl @@ -27,7 +27,7 @@ erlang_type/1, initial_capital/1, specs/1, suites/2, - specialized_specs/2, + test_categories/2, specialized_specs/2, subst_file/3, subst/2, print_data/1, make_non_erlang/2, maybe_atom_to_list/1, progress/4, @@ -96,26 +96,47 @@ specialized_specs(Dir,PostFix) -> Specs = filelib:wildcard(filename:join([filename:dirname(Dir), "*_test", "*_"++PostFix++".spec"])), sort_tests([begin - Base = filename:basename(Name), - list_to_atom(string:substr(Base,1,string:rstr(Base,"_")-1)) + DirPart = filename:dirname(Name), + AppTest = hd(lists:reverse(filename:split(DirPart))), + string:substr(AppTest, 1, length(AppTest)-5) end || Name <- Specs]). specs(Dir) -> Specs = filelib:wildcard(filename:join([filename:dirname(Dir), "*_test", "*.{dyn,}spec"])), - % Filter away all spec which end with {_bench,_smoke}.spec - NoBench = fun(SpecName) -> - case lists:reverse(SpecName) of - "ceps.hcneb_"++_ -> false; - "ceps.ekoms_"++_ -> false; - _ -> true - end - end, - - sort_tests([filename_to_atom(Name) || Name <- Specs, NoBench(Name)]). - -suites(Dir, Spec) -> - Glob=filename:join([filename:dirname(Dir), Spec++"_test", + %% Make sure only to include the main spec for each application + MainSpecs = + lists:flatmap(fun(FullName) -> + [Spec,TestDir|_] = + lists:reverse(filename:split(FullName)), + [_TestSuffix|TDParts] = + lists:reverse(string:tokens(TestDir,[$_,$.])), + [_SpecSuffix|SParts] = + lists:reverse(string:tokens(Spec,[$_,$.])), + if TDParts == SParts -> + [filename_to_atom(FullName)]; + true -> + [] + end + end, Specs), + sort_tests(MainSpecs). + +test_categories(Dir, App) -> + Specs = filelib:wildcard(filename:join([filename:dirname(Dir), + App++"_test", "*.spec"])), + lists:flatmap(fun(FullName) -> + [Spec,_TestDir|_] = + lists:reverse(filename:split(FullName)), + case filename:rootname(Spec -- App) of + "" -> + []; + [_Sep | Cat] -> + [list_to_atom(Cat)] + end + end, Specs). + +suites(Dir, App) -> + Glob=filename:join([filename:dirname(Dir), App++"_test", "*_SUITE.erl"]), Suites=filelib:wildcard(Glob), [filename_to_atom(Name) || Name <- Suites]. |