aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--lib/common_test/doc/src/run_test_chapter.xml25
-rw-r--r--lib/common_test/src/ct.erl45
-rw-r--r--lib/common_test/src/ct_framework.erl24
-rw-r--r--lib/common_test/src/ct_run.erl27
-rw-r--r--lib/common_test/src/ct_testspec.erl34
-rw-r--r--lib/common_test/src/ct_util.hrl1
-rw-r--r--lib/test_server/src/ts.erl557
-rw-r--r--lib/test_server/src/ts_lib.erl53
8 files changed, 509 insertions, 257 deletions
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 864f82cb63..df60e5f7f2 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -1005,6 +1005,31 @@
for starting the tests, the relaxed scanner
mode is enabled by means of the tuple: <c>{allow_user_terms,true}</c></p>
</section>
+ <section>
+ <title>Reading test specification terms</title>
+ <p>It's possible to look up terms in the current test specification
+ (i.e. the spec that's been used to configure and run the current test).
+ The function <c>get_testspec_terms()</c> returns a list of all test spec
+ terms (both config- and test terms) and <c>get_testspec_terms(Tags)</c>
+ returns the term (or a list of terms) matching the tag (or tags) in
+ <c>Tags</c>.</p>
+ <p>For example, in the test specification:</p>
+ <pre>
+ ...
+ {label, my_server_smoke_test}.
+ {config, "../../my_server_setup.cfg"}.
+ {config, "../../my_server_interface.cfg"}.
+ ...</pre>
+ <p>And in e.g. a test suite or a CT hook function:</p>
+ <pre>
+ ...
+ [{label,[{_Node,TestType}]}, {config,CfgFiles}] =
+ ct:get_testspec_terms([label,config]),
+
+ [verify_my_server_cfg(TestType, CfgFile) || {Node,CfgFile} &lt;- CfgFiles,
+ Node == node()];
+ ...</pre>
+ </section>
</section>
<section>
diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 9d8fce2789..5ed1346f1e 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -79,6 +79,7 @@
%% Other interface functions
-export([get_status/0, abort_current_testcase/1,
get_event_mgr_ref/0,
+ get_testspec_terms/0, get_testspec_terms/1,
encrypt_config_file/2, encrypt_config_file/3,
decrypt_config_file/2, decrypt_config_file/3]).
@@ -463,6 +464,50 @@ reload_config(Required)->
ct_config:reload_config(Required).
%%%-----------------------------------------------------------------
+%%% @spec get_testspec_terms() -> TestSpecTerms | undefined
+%%% TestSpecTerms = [{Tag,Value}]
+%%% Value = [term()]
+%%%
+%%% @doc Get a list of all test specification terms used to
+%%% configure and run this test.
+%%%
+get_testspec_terms() ->
+ case ct_util:get_testdata(testspec) of
+ undefined ->
+ undefined;
+ CurrSpecRec ->
+ ct_testspec:testspec_rec2list(CurrSpecRec)
+ end.
+
+%%%-----------------------------------------------------------------
+%%% @spec get_testspec_terms(Tags) -> TestSpecTerms | undefined
+%%% Tags = [Tag] | Tag
+%%% Tag = atom()
+%%% TestSpecTerms = [{Tag,Value}] | {Tag,Value}
+%%% Value = [{Node,term()}] | [term()]
+%%% Node = atom()
+%%%
+%%% @doc Read one or more terms from the test specification used
+%%% to configure and run this test. Tag is any valid test specification
+%%% tag, such as e.g. <c>label</c>, <c>config</c>, <c>logdir</c>.
+%%% User specific terms are also available to read if the
+%%% <c>allow_user_terms</c> option has been set. Note that all value tuples
+%%% returned, except user terms, will have the node name as first element.
+%%% Note also that in order to read test terms, use <c>Tag = tests</c>
+%%% (rather than <c>suites</c>, <c>groups</c> or <c>cases</c>). Value is
+%%% then the list of *all* tests on the form:
+%%% <c>[{Node,Dir,[{TestSpec,GroupsAndCases1},...]},...], where
+%%% GroupsAndCases = [{Group,[Case]}] | [Case]</c>.
+get_testspec_terms(Tags) ->
+ case ct_util:get_testdata(testspec) of
+ undefined ->
+ undefined;
+ CurrSpecRec ->
+ ct_testspec:testspec_rec2list(Tags, CurrSpecRec)
+ end.
+
+
+%%%-----------------------------------------------------------------
%%% @spec log(Format) -> ok
%%% @equiv log(default,50,Format,[])
log(Format) ->
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index ea3d7c8218..6ac16fef4d 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1062,14 +1062,32 @@ get_all_cases1(_, []) ->
get_all(Mod, ConfTests) ->
case catch apply(Mod, all, []) of
- {'EXIT',_} ->
+ {'EXIT',{undef,[{Mod,all,[],_} | _]}} ->
+ Reason = list_to_atom(atom_to_list(Mod)++":all/0 is missing"),
+ %% this makes test_server call error_in_suite as first
+ %% (and only) test case so we can report Reason properly
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {'EXIT',ExitReason} ->
Reason =
case code:which(Mod) of
non_existing ->
list_to_atom(atom_to_list(Mod)++
- " can not be compiled or loaded");
+ " can not be compiled or loaded");
_ ->
- list_to_atom(atom_to_list(Mod)++":all/0 is missing")
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:all/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(user, ErrStr, []),
+ %% save the error info so it doesn't
+ %% get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ list_to_atom(atom_to_list(Mod)++":all/0 failed")
end,
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Reason properly
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 4d74fd6a80..f6afa423cc 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -77,7 +77,8 @@
multiply_timetraps = 1,
scale_timetraps = false,
create_priv_dir,
- testspecs = [],
+ testspec_files = [],
+ current_testspec,
tests,
starter}).
@@ -485,8 +486,11 @@ execute_one_spec(TS, Opts, Args) ->
case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
ok -> % read tests from spec
{Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, Opts#opts{config=AllConfig,
- logdir=TheLogDir}, Args);
+ Result = do_run(Run, Skip, Opts#opts{config=AllConfig,
+ logdir=TheLogDir,
+ current_testspec=TS}, Args),
+ ct_util:delete_testdata(testspec),
+ Result;
Error ->
Error
end.
@@ -577,7 +581,7 @@ combine_test_opts(TS, Specs, Opts) ->
Opts#opts{label = Label,
profile = Profile,
- testspecs = Specs,
+ testspec_files = Specs,
cover = Cover,
cover_stop = CoverStop,
logdir = which(logdir, LogDir),
@@ -702,7 +706,7 @@ script_start4(#opts{label = Label, profile = Profile,
logopts = LogOpts,
verbosity = Verbosity,
enable_builtin_hooks = EnableBuiltinHooks,
- logdir = LogDir, testspecs = Specs}, _Args) ->
+ logdir = LogDir, testspec_files = Specs}, _Args) ->
%% label - used by ct_logs
application:set_env(common_test, test_label, Label),
@@ -1103,7 +1107,7 @@ run_test2(StartOpts) ->
undefined ->
case lists:keysearch(prepared_tests, 1, StartOpts) of
{value,{_,{Run,Skip},Specs}} -> % use prepared tests
- run_prepared(Run, Skip, Opts#opts{testspecs = Specs},
+ run_prepared(Run, Skip, Opts#opts{testspec_files = Specs},
StartOpts);
false ->
run_dir(Opts, StartOpts)
@@ -1111,11 +1115,11 @@ run_test2(StartOpts) ->
Specs ->
Relaxed = get_start_opt(allow_user_terms, value, false, StartOpts),
%% using testspec(s) as input for test
- run_spec_file(Relaxed, Opts#opts{testspecs = Specs}, StartOpts)
+ run_spec_file(Relaxed, Opts#opts{testspec_files = Specs}, StartOpts)
end.
run_spec_file(Relaxed,
- Opts = #opts{testspecs = Specs},
+ Opts = #opts{testspec_files = Specs},
StartOpts) ->
Specs1 = case Specs of
[X|_] when is_integer(X) -> [Specs];
@@ -1399,7 +1403,7 @@ run_testspec2(TestSpec) ->
case check_and_install_configfiles(
Opts#opts.config, LogDir1, Opts) of
ok ->
- Opts1 = Opts#opts{testspecs = [],
+ Opts1 = Opts#opts{testspec_files = [],
logdir = LogDir1,
include = AllInclude},
{Run,Skip} = ct_testspec:prepare_tests(TS, node()),
@@ -1706,6 +1710,9 @@ compile_and_run(Tests, Skip, Opts, Args) ->
ct_util:set_testdata({stylesheet,Opts#opts.stylesheet}),
%% save logopts
ct_util:set_testdata({logopts,Opts#opts.logopts}),
+ %% save info about current testspec (testspec record or undefined)
+ ct_util:set_testdata({testspec,Opts#opts.current_testspec}),
+
%% enable silent connections
case Opts#opts.silent_connections of
[] ->
@@ -1720,7 +1727,7 @@ compile_and_run(Tests, Skip, Opts, Args) ->
ct_logs:log("Silent connections", "~p", [Conns])
end
end,
- log_ts_names(Opts#opts.testspecs),
+ log_ts_names(Opts#opts.testspec_files),
TestSuites = suite_tuples(Tests),
{_TestSuites1,SuiteMakeErrors,AllMakeErrors} =
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 10a9bdac67..10c3f2a938 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -27,6 +27,8 @@
collect_tests_from_list/2, collect_tests_from_list/3,
collect_tests_from_file/2, collect_tests_from_file/3]).
+-export([testspec_rec2list/1, testspec_rec2list/2]).
+
-include("ct_util.hrl").
-define(testspec_fields, record_info(fields, testspec)).
@@ -973,7 +975,8 @@ add_tests([Term={Tag,all_nodes,Data}|Ts],Spec) ->
should_be_added(Tag,Node,Data,Spec)],
add_tests(Tests++Ts,Spec);
invalid -> % ignore term
- add_tests(Ts,Spec)
+ Unknown = Spec#testspec.unknown,
+ add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]})
end;
%% create one test entry per node in Nodes and reinsert
add_tests([{Tag,[],Data}|Ts],Spec) ->
@@ -1001,7 +1004,8 @@ add_tests([Term={Tag,NodeOrOther,Data}|Ts],Spec) ->
handle_data(Tag,Node,Data,Spec),
add_tests(Ts,mod_field(Spec,Tag,NodeIxData));
invalid -> % ignore term
- add_tests(Ts,Spec)
+ Unknown = Spec#testspec.unknown,
+ add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]})
end;
false ->
add_tests([{Tag,all_nodes,{NodeOrOther,Data}}|Ts],Spec)
@@ -1012,13 +1016,15 @@ add_tests([Term={Tag,Data}|Ts],Spec) ->
valid ->
add_tests([{Tag,all_nodes,Data}|Ts],Spec);
invalid ->
- add_tests(Ts,Spec)
+ Unknown = Spec#testspec.unknown,
+ add_tests(Ts,Spec#testspec{unknown=Unknown++[Term]})
end;
%% some other data than a tuple
add_tests([Other|Ts],Spec) ->
case get(relaxed) of
- true ->
- add_tests(Ts,Spec);
+ true ->
+ Unknown = Spec#testspec.unknown,
+ add_tests(Ts,Spec#testspec{unknown=Unknown++[Other]});
false ->
throw({error,{undefined_term_in_spec,Other}})
end;
@@ -1149,6 +1155,24 @@ per_node([N|Ns],Tag,Data,Refs) ->
per_node([],_,_,_) ->
[].
+%% Change the testspec record "back" to a list of tuples
+testspec_rec2list(Rec) ->
+ {Terms,_} = lists:mapfoldl(fun(unknown, Pos) ->
+ {element(Pos, Rec),Pos+1};
+ (F, Pos) ->
+ {{F,element(Pos, Rec)},Pos+1}
+ end,2,?testspec_fields),
+ lists:flatten(Terms).
+
+%% Extract one or more values from a testspec record and
+%% return the result as a list of tuples
+testspec_rec2list(Field, Rec) when is_atom(Field) ->
+ [Term] = testspec_rec2list([Field], Rec),
+ Term;
+testspec_rec2list(Fields, Rec) ->
+ Terms = testspec_rec2list(Rec),
+ [{Field,proplists:get_value(Field, Terms)} || Field <- Fields].
+
%% read the value for FieldName in record Rec#testspec
read_field(Rec, FieldName) ->
catch lists:foldl(fun(F, Pos) when F == FieldName ->
diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl
index 845bb55486..f4cf407856 100644
--- a/lib/common_test/src/ct_util.hrl
+++ b/lib/common_test/src/ct_util.hrl
@@ -55,6 +55,7 @@
create_priv_dir=[],
alias=[],
tests=[],
+ unknown=[],
merge_tests=true}).
-record(cover, {app=none,
diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl
index d6d2e865e2..c2a7afa38b 100644
--- a/lib/test_server/src/ts.erl
+++ b/lib/test_server/src/ts.erl
@@ -24,11 +24,11 @@
-module(ts).
--export([run/0, run/1, run/2, run/3, run/4, run/5,
- tests/0, tests/1,
+-export([cl_run/1,
+ run/0, run/1, run/2, run/3, run/4, run/5,
+ run_category/1, run_category/2, run_category/3,
+ tests/0, tests/1, suites/1, categories/1,
install/0, install/1,
- bench/0, bench/1, bench/2, benchmarks/0,
- smoke_test/0, smoke_test/1,smoke_test/2, smoke_tests/0,
estone/0, estone/1,
cross_cover_analyse/1,
compile_testcases/0, compile_testcases/1,
@@ -82,10 +82,13 @@
-define(
install_help,
[
- " ts:install() - Install TS with no Options.\n"
- " ts:install([Options]) - Install TS with Options\n"
+ " ts:install()\n",
+ " Install ts with no options.\n",
"\n",
- "Installation options supported:\n",
+ " ts:install(Options)\n",
+ " Install ts with a list of options, see below.\n",
+ "\n",
+ "Installation options supported:\n\n",
" {longnames, true} - Use fully qualified hostnames\n",
" {verbose, Level} - Sets verbosity level for TS output (0,1,2), 0 is\n"
" quiet(default).\n"
@@ -110,21 +113,64 @@ help() ->
end.
help(uninstalled) ->
- H = ["TS is not installed yet. To install use:\n\n"],
+ H = ["ts is not yet installed. To install use:\n\n"],
show_help([H,?install_help]);
help(installed) ->
- H = ["Run functions:\n",
- " ts:run() - Run all available tests.\n",
- " ts:run(Spec) - Run all tests in given test spec file.\n",
- " The spec file is actually ../*_test/Spec.spec\n",
- " ts:run([Specs]) - Run all tests in all given test spec files.\n",
- " ts:run(Spec, Mod) - Run a single test suite.\n",
- " ts:run(Spec, Mod, Case)\n",
- " - Run a single test case.\n",
- " All above run functions can have an additional Options argument\n",
- " which is a list of options.\n",
+ H = ["\n",
+ "Run functions:\n\n",
+ " ts:run()\n",
+ " Run the tests for all apps. The tests are defined by the\n",
+ " main test specification for each app: ../App_test/App.spec.\n",
+ "\n",
+ " ts:run(Apps)\n",
+ " Apps = atom() | [atom()]\n",
+ " Run the tests for an app, or set of apps. The tests are\n",
+ " defined by the main test specification for each app:\n",
+ " ../App_test/App.spec.\n",
+ "\n",
+ " ts:run(App, Suites)\n",
+ " App = atom(), Suites = atom() | [atom()]\n",
+ " Run one or more test suites for App (i.e. modules named\n",
+ " *_SUITE.erl, located in ../App_test/).\n",
+ "\n",
+ " ts:run(App, Suite, TestCases)\n",
+ " App = atom(), Suite = atom(),\n",
+ " TestCases = TCs | {testcase,TCs}, TCs = atom() | [atom()]\n",
+ " Run one or more test cases (functions) in Suite.\n",
+ "\n",
+ " ts:run(App, Suite, {group,Groups})\n",
+ " App = atom(), Suite = atom(), Groups = atom() | [atom()]\n",
+ " Run one or more test case groups in Suite.\n",
+ "\n",
+ " ts:run(App, Suite, {group,Group}, {testcase,TestCases})\n",
+ " App = atom(), Suite = atom(), Group = atom(),\n",
+ " TestCases = atom() | [atom()]\n",
+ " Run one or more test cases in a test case group in Suite.\n",
+ "\n",
+ " ts:run_category(TestCategory)\n",
+ " TestCategory = smoke | essential | bench | atom()\n",
+ " Run the specified category of tests for all apps.\n",
+ " For each app, the tests are defined by the specification:\n",
+ " ../App_test/App_TestCategory.spec.\n",
+ "\n",
+ " ts:run_category(Apps, TestCategory)\n",
+ " Apps = atom() | [atom()],\n",
+ " TestCategory = smoke | essential | bench | atom()\n",
+ " Run the specified category of tests for the given app or apps.\n",
"\n",
- "Run options supported:\n",
+ " Note that the test category parameter may have arbitrary value,\n",
+ " but should correspond to an existing test specification with file\n",
+ " name: ../App_test/App_TestCategory.spec.\n",
+ " Predefined categories exist for smoke tests, essential tests and\n",
+ " benchmark tests. The corresponding specs are:\n",
+ " ../*_test/Spec_smoke.spec, ../*_test/Spec_essential.spec and\n",
+ " ../*_test/Spec_bench.spec.\n",
+ "\n",
+ " All above run functions can take an additional last argument,\n",
+ " Options, which is a list of options (e.g. ts:run(App, Options),\n",
+ " or ts:run_category(Apps, TestCategory, Options)).\n",
+ "\n",
+ "Run options supported:\n\n",
" batch - Do not start a new xterm\n",
" {verbose, Level} - Same as the verbosity option for install\n",
" verbose - Same as {verbose, 1}\n",
@@ -143,47 +189,46 @@ help(installed) ->
" files are. The default location is\n"
" tests/test_server/.\n"
"\n",
- "Supported trace information elements\n",
+ "Supported trace information elements:\n\n",
" {tp | tpl, Mod, [] | match_spec()}\n",
" {tp | tpl, Mod, Func, [] | match_spec()}\n",
" {tp | tpl, Mod, Func, Arity, [] | match_spec()}\n",
" {ctp | ctpl, Mod}\n",
" {ctp | ctpl, Mod, Func}\n",
" {ctp | ctpl, Mod, Func, Arity}\n",
+ "\n\n",
+ "Support functions:\n\n",
+ " ts:tests()\n",
+ " Returns all apps available for testing.\n",
+ "\n",
+ " ts:tests(TestCategory)\n",
+ " Returns all apps that provide tests in the given category.\n",
+ "\n",
+ " ts:suites(App)\n",
+ " Returns all available test suites for App,\n",
+ " i.e. ../App_test/*_SUITE.erl\n",
+ "\n",
+ " ts:categories(App)\n",
+ " Returns all test categories available for App.\n",
+ "\n",
+ " ts:estone()\n",
+ " Runs estone_SUITE in the kernel application with no run options\n",
+ "\n",
+ " ts:estone(Opts)\n",
+ " Runs estone_SUITE in the kernel application with the given\n",
+ " run options\n",
+ "\n",
+ " ts:cross_cover_analyse(Level)\n",
+ " Use after ts:run with option cover or cover_details. Analyses\n",
+ " modules specified with a 'cross' statement in the cover spec file.\n",
+ " Level can be 'overview' or 'details'.\n",
"\n",
- "Support functions:\n",
- " ts:tests() - Shows all available families of tests.\n",
- " ts:tests(Spec) - Shows all available test modules in Spec,\n",
- " i.e. ../Spec_test/*_SUITE.erl\n",
- " ts:estone() - Run estone_SUITE in kernel application with\n"
- " no run options\n",
- " ts:estone(Opts) - Run estone_SUITE in kernel application with\n"
- " the given run options\n",
- " ts:cross_cover_analyse(Level)\n"
- " - Used after ts:run with option cover or \n"
- " cover_details. Analyses modules specified with\n"
- " a 'cross' statement in the cover spec file.\n"
- " Level can be 'overview' or 'details'.\n",
- " ts:compile_testcases()~n"
- " ts:compile_testcases(Apps)~n"
- " - Compile all testcases for usage in a cross ~n"
- " compile environment."
- " \n"
- "Benchmark functions:\n"
- " ts:benchmarks() - Get all available families of benchmarks\n"
- " ts:bench() - Runs all benchmarks\n"
- " ts:bench(Spec) - Runs all benchmarks in the given spec file.\n"
- " The spec file is actually ../*_test/Spec_bench.spec\n\n"
- " ts:bench can take the same Options argument as ts:run.\n"
- "Smoke test functions:\n"
- " ts:smoke_tests() - Get all available families of smoke tests\n"
- " ts:smoke_test() - Runs all smoke tests\n"
- " ts:smoke_test(Spec)\n"
- " - Runs all smoke tests in the given spec file.\n"
- " The spec file is actually ../*_test/Spec_smoke.spec\n\n"
- " ts:smoke_test can take the same Options argument as ts:run.\n"
- "\n"
- "Installation (already done):\n"
+ " ts:compile_testcases()\n",
+ " ts:compile_testcases(Apps)\n",
+ " Compiles all test cases for the given apps, for usage in a\n",
+ " cross compilation environment.\n",
+ "\n\n",
+ "Installation (already done):\n\n"
],
show_help([H,?install_help]).
@@ -212,86 +257,128 @@ run_all(_Vars) ->
run_some([], _Opts) ->
ok;
-run_some([{Spec,Mod}|Specs], Opts) ->
- case run(Spec, Mod, Opts) of
+run_some([{App,Mod}|Apps], Opts) ->
+ case run(App, Mod, Opts) of
ok -> ok;
- Error -> io:format("~p: ~p~n",[{Spec,Mod},Error])
+ Error -> io:format("~p: ~p~n",[{App,Mod},Error])
end,
- run_some(Specs, Opts);
-run_some([Spec|Specs], Opts) ->
- case run(Spec, Opts) of
+ run_some(Apps, Opts);
+run_some([App|Apps], Opts) ->
+ case run(App, Opts) of
ok -> ok;
- Error -> io:format("~p: ~p~n",[Spec,Error])
+ Error -> io:format("~p: ~p~n",[App,Error])
end,
- run_some(Specs, Opts).
+ run_some(Apps, Opts).
+
+%% This can be used from command line. Both App and
+%% TestCategory must be specified. App may be 'all'
+%% and TestCategory may be 'main'. Examples:
+%% erl -s ts cl_run kernel smoke <options>
+%% erl -s ts cl_run kernel main <options>
+%% erl -s ts cl_run all essential <options>
+%% erl -s ts cl_run all main <options>
+%% When using the 'main' category and running with cover,
+%% one can also use the cross_cover_analysis flag.
+cl_run([App,Cat|Options0]) when is_atom(App) ->
-%% Runs one test spec (interactive).
-run(Testspec) when is_atom(Testspec) ->
- Options=check_test_get_opts(Testspec, []),
- File = atom_to_list(Testspec),
- run_test(File, [{spec,[File++".spec"]}], Options);
-
-%% This can be used from command line, e.g.
-%% erl -s ts run all_tests <config>
-%% When using the all_tests flag and running with cover, one can also
-%% use the cross_cover_analysis flag.
-run([all_tests|Config0]) ->
AllAtomsFun = fun(X) when is_atom(X) -> true;
(_) -> false
end,
- Config1 =
- case lists:all(AllAtomsFun,Config0) of
+ Options1 =
+ case lists:all(AllAtomsFun, Options0) of
true ->
%% Could be from command line
- lists:map(fun(Conf)->to_erlang_term(Conf) end,Config0)--[batch];
+ lists:map(fun(Opt) ->
+ to_erlang_term(Opt)
+ end, Options0) -- [batch];
false ->
- Config0--[batch]
+ Options0 -- [batch]
end,
%% Make sure there is exactly one occurence of 'batch'
- Config2 = [batch|Config1],
-
- R = run(tests(),Config2),
-
- case check_for_cross_cover_analysis_flag(Config2) of
+ Options2 = [batch|Options1],
+
+ Result =
+ case {App,Cat} of
+ {all,main} ->
+ run(tests(), Options2);
+ {all,Cat} ->
+ run_category(Cat, Options2);
+ {_,main} ->
+ run(App, Options2);
+ {_,Cat} ->
+ run_category(App, Cat, Options2)
+ end,
+ case check_for_cross_cover_analysis_flag(Options2) of
false ->
ok;
Level ->
cross_cover_analyse(Level)
end,
+ Result.
- R;
-
-%% ts:run(ListOfTests)
-run(List) when is_list(List) ->
- run(List, [batch]).
+%% run/1
+%% Runs tests for one app (interactive).
+run(App) when is_atom(App) ->
+ Options = check_test_get_opts(App, []),
+ File = atom_to_list(App),
+ run_test(File, [{spec,[File++".spec"]},{allow_user_terms,true}], Options);
-run(List, Opts) when is_list(List), is_list(Opts) ->
- run_some(List, Opts);
+%% This can be used from command line, e.g.
+%% erl -s ts run all <options>
+%% erl -s ts run main <options>
+run([all,main|Opts]) ->
+ cl_run([all,main|Opts]);
+run([all|Opts]) ->
+ cl_run([all,main|Opts]);
+run([main|Opts]) ->
+ cl_run([all,main|Opts]);
+%% Backwards compatible
+run([all_tests|Opts]) ->
+ cl_run([all,main|Opts]);
+
+%% run/1
+%% Runs the main tests for all available apps
+run(Apps) when is_list(Apps) ->
+ run(Apps, [batch]).
%% run/2
-%% Runs one test spec with list of suites or with options
-run(Testspec, ModsOrConfig) when is_atom(Testspec),
- is_list(ModsOrConfig) ->
- case is_list_of_suites(ModsOrConfig) of
+%% Runs the main tests for all available apps
+run(Apps, Opts) when is_list(Apps), is_list(Opts) ->
+ run_some(Apps, Opts);
+
+%% Runs tests for one app with list of suites or with options
+run(App, ModsOrOpts) when is_atom(App),
+ is_list(ModsOrOpts) ->
+ case is_list_of_suites(ModsOrOpts) of
false ->
- run(Testspec, {config_list,ModsOrConfig});
+ run(App, {opts_list,ModsOrOpts});
true ->
- run_some([{Testspec,M} || M <- ModsOrConfig],
+ run_some([{App,M} || M <- ModsOrOpts],
[batch])
end;
-run(Testspec, {config_list,Config}) ->
- Options=check_test_get_opts(Testspec, Config),
- IsSmoke=proplists:get_value(smoke,Config),
- File=atom_to_list(Testspec),
+
+run(App, {opts_list,Opts}) ->
+ Options = check_test_get_opts(App, Opts),
+ File = atom_to_list(App),
+
+ %% check if other test category than main has been specified
+ {CatSpecName,TestCat} =
+ case proplists:get_value(test_category, Opts) of
+ undefined ->
+ {"",main};
+ Cat ->
+ {"_" ++ atom_to_list(Cat),Cat}
+ end,
+
WhatToDo =
- case Testspec of
+ case App of
%% Known to exist but fails generic tests below
emulator -> test;
system -> test;
erl_interface -> test;
epmd -> test;
_ ->
- case code:lib_dir(Testspec) of
+ case code:lib_dir(App) of
{error,bad_name} ->
%% Application does not exist
skip;
@@ -313,92 +400,135 @@ run(Testspec, {config_list,Config}) ->
end
end
end,
- Spec =
- case WhatToDo of
- skip ->
- create_skip_spec(Testspec, tests(Testspec));
- test when IsSmoke ->
- File++"_smoke.spec";
- test ->
- File++".spec"
- end,
- run_test(File, [{spec,[Spec]}], Options);
-%% Runs one module in a spec (interactive)
-run(Testspec, Mod) when is_atom(Testspec), is_atom(Mod) ->
- run_test({atom_to_list(Testspec),Mod},
+ case WhatToDo of
+ skip ->
+ SkipSpec = create_skip_spec(App, suites(App)),
+ run_test(File, [{spec,[SkipSpec]}], Options);
+ test when TestCat == bench ->
+ check_and_run(fun(Vars) ->
+ ts_benchmark:run([App], Options, Vars)
+ end);
+ test ->
+ Spec = File ++ CatSpecName ++ ".spec",
+ run_test(File, [{spec,[Spec]},{allow_user_terms,true}], Options)
+ end;
+
+%% Runs one module for an app (interactive)
+run(App, Mod) when is_atom(App), is_atom(Mod) ->
+ run_test({atom_to_list(App),Mod},
[{suite,Mod}],
[interactive]).
%% run/3
-%% Run one module in a spec with Config
-run(Testspec, Mod, Config) when is_atom(Testspec),
- is_atom(Mod),
- is_list(Config) ->
- Options=check_test_get_opts(Testspec, Config),
- run_test({atom_to_list(Testspec),Mod},
+%% Run one module for an app with Opts
+run(App, Mod, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
+ run_test({atom_to_list(App),Mod},
[{suite,Mod}], Options);
-%% Run multiple modules with Config
-run(Testspec, Mods, Config) when is_atom(Testspec),
- is_list(Mods),
- is_list(Config) ->
- run_some([{Testspec,M} || M <- Mods], Config);
+
+%% Run multiple modules with Opts
+run(App, Mods, Opts) when is_atom(App),
+ is_list(Mods),
+ is_list(Opts) ->
+ run_some([{App,M} || M <- Mods], Opts);
+
%% Runs one test case in a module.
-run(Testspec, Mod, Case) when is_atom(Testspec),
- is_atom(Mod),
- is_atom(Case) ->
- Options=check_test_get_opts(Testspec, []),
+run(App, Mod, Case) when is_atom(App),
+ is_atom(Mod),
+ is_atom(Case) ->
+ Options = check_test_get_opts(App, []),
Args = [{suite,Mod},{testcase,Case}],
- run_test(atom_to_list(Testspec), Args, Options);
+ run_test(atom_to_list(App), Args, Options);
+
%% Runs one or more groups in a module.
-run(Testspec, Mod, Grs={group,_Groups}) when is_atom(Testspec),
- is_atom(Mod) ->
- Options=check_test_get_opts(Testspec, []),
+run(App, Mod, Grs={group,_Groups}) when is_atom(App),
+ is_atom(Mod) ->
+ Options = check_test_get_opts(App, []),
Args = [{suite,Mod},Grs],
- run_test(atom_to_list(Testspec), Args, Options);
+ run_test(atom_to_list(App), Args, Options);
+
%% Runs one or more test cases in a module.
-run(Testspec, Mod, TCs={testcase,_Cases}) when is_atom(Testspec),
- is_atom(Mod) ->
- Options=check_test_get_opts(Testspec, []),
+run(App, Mod, TCs={testcase,_Cases}) when is_atom(App),
+ is_atom(Mod) ->
+ Options = check_test_get_opts(App, []),
Args = [{suite,Mod},TCs],
- run_test(atom_to_list(Testspec), Args, Options).
+ run_test(atom_to_list(App), Args, Options).
%% run/4
%% Run one test case in a module with Options.
-run(Testspec, Mod, Case, Config) when is_atom(Testspec),
- is_atom(Mod),
- is_atom(Case),
- is_list(Config) ->
- Options=check_test_get_opts(Testspec, Config),
+run(App, Mod, Case, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_atom(Case),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
Args = [{suite,Mod},{testcase,Case}],
- run_test(atom_to_list(Testspec), Args, Options);
+ run_test(atom_to_list(App), Args, Options);
+
%% Run one or more test cases in a module with Options.
-run(Testspec, Mod, {testcase,Cases}, Config) when is_atom(Testspec),
- is_atom(Mod) ->
- run(Testspec, Mod, Cases, Config);
-run(Testspec, Mod, Cases, Config) when is_atom(Testspec),
- is_atom(Mod),
- is_list(Cases),
- is_list(Config) ->
- Options=check_test_get_opts(Testspec, Config),
+run(App, Mod, {testcase,Cases}, Opts) when is_atom(App),
+ is_atom(Mod) ->
+ run(App, Mod, Cases, Opts);
+run(App, Mod, Cases, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Cases),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
Args = [{suite,Mod},Cases],
- run_test(atom_to_list(Testspec), Args, Options);
+ run_test(atom_to_list(App), Args, Options);
+
+%% Run one or more test cases in a group.
+run(App, Mod, Gr={group,_Group}, {testcase,Cases}) when is_atom(App),
+ is_atom(Mod) ->
+ run(App, Mod, Gr, Cases, [batch]);
+
+
%% Run one or more groups in a module with Options.
-run(Testspec, Mod, Grs={group,_Groups}, Config) when is_atom(Testspec),
- is_atom(Mod) ->
- Options=check_test_get_opts(Testspec, Config),
+run(App, Mod, Grs={group,_Groups}, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
Args = [{suite,Mod},Grs],
- run_test(atom_to_list(Testspec), Args, Options).
+ run_test(atom_to_list(App), Args, Options).
%% run/5
%% Run one or more test cases in a group with Options.
-run(Testspec, Mod, Group, Cases, Config) when is_atom(Testspec),
- is_atom(Mod),
- is_list(Config) ->
+run(App, Mod, Group, Cases, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Opts) ->
Group1 = if is_tuple(Group) -> Group; true -> {group,Group} end,
Cases1 = if is_tuple(Cases) -> Cases; true -> {testcase,Cases} end,
- Options=check_test_get_opts(Testspec, Config),
+ Options = check_test_get_opts(App, Opts),
Args = [{suite,Mod},Group1,Cases1],
- run_test(atom_to_list(Testspec), Args, Options).
+ run_test(atom_to_list(App), Args, Options).
+
+%% run_category/1
+run_category(TestCategory) when is_atom(TestCategory) ->
+ run_category(TestCategory, [batch]).
+
+%% run_category/2
+run_category(TestCategory, Opts) when is_atom(TestCategory),
+ is_list(Opts) ->
+ case ts:tests() of
+ [] ->
+ {error, no_tests_available};
+ Apps ->
+ Opts1 = [{test_category,TestCategory} | Opts],
+ run_some(Apps, Opts1)
+ end;
+
+run_category(Apps, TestCategory) when is_atom(TestCategory) ->
+ run_category(Apps, TestCategory, [batch]).
+
+%% run_category/3
+run_category(App, TestCategory, Opts) ->
+ Apps = if is_atom(App) -> [App];
+ is_list(App) -> App
+ end,
+ Opts1 = [{test_category,TestCategory} | Opts],
+ run_some(Apps, Opts1).
+
is_list_of_suites(List) ->
lists:all(fun(Suite) ->
@@ -416,29 +546,29 @@ is_list_of_suites(List) ->
%% Create a spec to skip all SUITES, this is used when the application
%% to be tested is not part of the OTP release to be tested.
-create_skip_spec(Testspec, SuitesToSkip) ->
+create_skip_spec(App, SuitesToSkip) ->
{ok,Cwd} = file:get_cwd(),
- TestspecString = atom_to_list(Testspec),
- Specname = TestspecString++"_skip.spec",
+ AppString = atom_to_list(App),
+ Specname = AppString++"_skip.spec",
{ok,D} = file:open(filename:join([filename:dirname(Cwd),
- TestspecString++"_test",Specname]),
+ AppString++"_test",Specname]),
[write]),
- TestDir = "\"../"++TestspecString++"_test\"",
+ TestDir = "\"../"++AppString++"_test\"",
io:format(D,"{suites, "++TestDir++", all}.~n",[]),
io:format(D,"{skip_suites, "++TestDir++", ~w, \"Skipped as application"
" is not in path!\"}.",[SuitesToSkip]),
Specname.
-%% Check testspec to be valid and get possible Options
-%% from the config.
-check_test_get_opts(Testspec, Config) ->
- validate_test(Testspec),
- Mode = configmember(batch, {batch, interactive}, Config),
- Vars = configvars(Config),
- Trace = get_config(trace,Config),
- ConfigPath = get_config(config,Config),
- KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Config),
- Cover = configcover(Testspec,Config),
+%% Check testspec for App to be valid and get possible options
+%% from the list.
+check_test_get_opts(App, Opts) ->
+ validate_test(App),
+ Mode = configmember(batch, {batch, interactive}, Opts),
+ Vars = configvars(Opts),
+ Trace = get_config(trace,Opts),
+ ConfigPath = get_config(config,Opts),
+ KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Opts),
+ Cover = configcover(App,Opts),
lists:flatten([Vars,Mode,Trace,KeepTopcase,Cover,ConfigPath]).
to_erlang_term(Atom) ->
@@ -447,7 +577,7 @@ to_erlang_term(Atom) ->
{ok, Term} = erl_parse:parse_term(Tokens),
Term.
-%% Validate that a Testspec really is a testspec,
+%% Validate that Testspec really is a testspec,
%% and exit if not.
validate_test(Testspec) ->
case lists:member(Testspec, tests()) of
@@ -460,10 +590,10 @@ validate_test(Testspec) ->
exit(self(), {error, test_not_available})
end.
-configvars(Config) ->
- case lists:keysearch(vars, 1, Config) of
+configvars(Opts) ->
+ case lists:keysearch(vars, 1, Opts) of
{value, {vars, List}} ->
- List0 = special_vars(Config),
+ List0 = special_vars(Opts),
Key = fun(T) -> element(1,T) end,
DelDupList =
lists:filter(fun(V) ->
@@ -474,17 +604,17 @@ configvars(Config) ->
end, List),
{vars, [List0|DelDupList]};
_ ->
- {vars, special_vars(Config)}
+ {vars, special_vars(Opts)}
end.
-%% Allow some shortcuts in the Options...
-special_vars(Config) ->
+%% Allow some shortcuts in the options...
+special_vars(Opts) ->
SpecVars =
- case lists:member(verbose, Config) of
+ case lists:member(verbose, Opts) of
true ->
[{verbose, 1}];
false ->
- case lists:keysearch(verbose, 1, Config) of
+ case lists:keysearch(verbose, 1, Opts) of
{value, {verbose, Lvl}} ->
[{verbose, Lvl}];
_ ->
@@ -492,13 +622,13 @@ special_vars(Config) ->
end
end,
SpecVars1 =
- case lists:keysearch(diskless, 1, Config) of
+ case lists:keysearch(diskless, 1, Opts) of
{value,{diskless, true}} ->
[{diskless, true} | SpecVars];
_ ->
SpecVars
end,
- case lists:keysearch(testcase_callback, 1, Config) of
+ case lists:keysearch(testcase_callback, 1, Opts) of
{value,{testcase_callback, CBM, CBF}} ->
[{ts_testcase_callback, {CBM,CBF}} | SpecVars1];
{value,{testcase_callback, CB}} ->
@@ -566,50 +696,31 @@ check_for_cross_cover_analysis_flag([_|Config],Level,CrossFlag) ->
check_for_cross_cover_analysis_flag([],_,_) ->
false.
-%% Returns a list of available test suites.
+%% Returns all available apps.
tests() ->
{ok, Cwd} = file:get_cwd(),
ts_lib:specs(Cwd).
-tests(Spec) ->
+%% Returns all apps that provide tests in the given test category
+tests(main) ->
{ok, Cwd} = file:get_cwd(),
- ts_lib:suites(Cwd, atom_to_list(Spec)).
-
-%% Benchmark related functions
-
-bench() ->
- bench([]).
-
-bench(Opts) when is_list(Opts) ->
- bench(benchmarks(),Opts);
-bench(Spec) ->
- bench([Spec],[]).
-
-bench(Spec, Opts) when is_atom(Spec) ->
- bench([Spec],Opts);
-bench(Specs, Opts) ->
- check_and_run(fun(Vars) -> ts_benchmark:run(Specs, Opts, Vars) end).
-
-benchmarks() ->
- ts_benchmark:benchmarks().
-
-smoke_test() ->
- smoke_test([]).
-
-smoke_test(Opts) when is_list(Opts) ->
- smoke_test(smoke_tests(),Opts);
-smoke_test(Spec) ->
- smoke_test([Spec],[]).
-
-smoke_test(Spec, Opts) when is_atom(Spec) ->
- smoke_test([Spec],Opts);
-smoke_test(Specs, Opts) ->
- run(Specs, [{smoke,true}|Opts]).
+ ts_lib:specs(Cwd);
+tests(bench) ->
+ ts_benchmark:benchmarks();
+tests(TestCategory) ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:specialized_specs(Cwd, atom_to_list(TestCategory)).
+
+%% Returns a list of available test suites for App.
+suites(App) ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:suites(Cwd, atom_to_list(App)).
-smoke_tests() ->
+%% Returns all available test categories for App
+categories(App) ->
{ok, Cwd} = file:get_cwd(),
- ts_lib:specialized_specs(Cwd,"smoke").
+ ts_lib:test_categories(Cwd, atom_to_list(App)).
%%
%% estone/0, estone/1
diff --git a/lib/test_server/src/ts_lib.erl b/lib/test_server/src/ts_lib.erl
index 5368960446..45746c444d 100644
--- a/lib/test_server/src/ts_lib.erl
+++ b/lib/test_server/src/ts_lib.erl
@@ -27,7 +27,7 @@
erlang_type/1,
initial_capital/1,
specs/1, suites/2,
- specialized_specs/2,
+ test_categories/2, specialized_specs/2,
subst_file/3, subst/2, print_data/1,
make_non_erlang/2,
maybe_atom_to_list/1, progress/4,
@@ -96,26 +96,47 @@ specialized_specs(Dir,PostFix) ->
Specs = filelib:wildcard(filename:join([filename:dirname(Dir),
"*_test", "*_"++PostFix++".spec"])),
sort_tests([begin
- Base = filename:basename(Name),
- list_to_atom(string:substr(Base,1,string:rstr(Base,"_")-1))
+ DirPart = filename:dirname(Name),
+ AppTest = hd(lists:reverse(filename:split(DirPart))),
+ string:substr(AppTest, 1, length(AppTest)-5)
end || Name <- Specs]).
specs(Dir) ->
Specs = filelib:wildcard(filename:join([filename:dirname(Dir),
"*_test", "*.{dyn,}spec"])),
- % Filter away all spec which end with {_bench,_smoke}.spec
- NoBench = fun(SpecName) ->
- case lists:reverse(SpecName) of
- "ceps.hcneb_"++_ -> false;
- "ceps.ekoms_"++_ -> false;
- _ -> true
- end
- end,
-
- sort_tests([filename_to_atom(Name) || Name <- Specs, NoBench(Name)]).
-
-suites(Dir, Spec) ->
- Glob=filename:join([filename:dirname(Dir), Spec++"_test",
+ %% Make sure only to include the main spec for each application
+ MainSpecs =
+ lists:flatmap(fun(FullName) ->
+ [Spec,TestDir|_] =
+ lists:reverse(filename:split(FullName)),
+ [_TestSuffix|TDParts] =
+ lists:reverse(string:tokens(TestDir,[$_,$.])),
+ [_SpecSuffix|SParts] =
+ lists:reverse(string:tokens(Spec,[$_,$.])),
+ if TDParts == SParts ->
+ [filename_to_atom(FullName)];
+ true ->
+ []
+ end
+ end, Specs),
+ sort_tests(MainSpecs).
+
+test_categories(Dir, App) ->
+ Specs = filelib:wildcard(filename:join([filename:dirname(Dir),
+ App++"_test", "*.spec"])),
+ lists:flatmap(fun(FullName) ->
+ [Spec,_TestDir|_] =
+ lists:reverse(filename:split(FullName)),
+ case filename:rootname(Spec -- App) of
+ "" ->
+ [];
+ [_Sep | Cat] ->
+ [list_to_atom(Cat)]
+ end
+ end, Specs).
+
+suites(Dir, App) ->
+ Glob=filename:join([filename:dirname(Dir), App++"_test",
"*_SUITE.erl"]),
Suites=filelib:wildcard(Glob),
[filename_to_atom(Name) || Name <- Suites].