aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test_server/ts.erl
diff options
context:
space:
mode:
authorBjörn Gustavsson <[email protected]>2016-02-16 06:45:27 +0100
committerBjörn Gustavsson <[email protected]>2016-02-17 10:35:22 +0100
commitdcda9b507bf14391c8bed91bfa9c56355342b681 (patch)
tree45658baf8d18d7f363a3044972201b957acaffb1 /lib/common_test/test_server/ts.erl
parent40aaa8bfa8ec0776a4d70079cf34a5bd337d42fe (diff)
downloadotp-dcda9b507bf14391c8bed91bfa9c56355342b681.tar.gz
otp-dcda9b507bf14391c8bed91bfa9c56355342b681.tar.bz2
otp-dcda9b507bf14391c8bed91bfa9c56355342b681.zip
Remove test_server as a standalone application
The test_server application has previously been deprecated. In OTP 19, we will move relevant parts of test_server into the common_test application. Test suites that include test_server.hrl must be updated to include ct.hrl instead. Test suites that include test_server_line.hrl must removed that inclusion. Test suites that call the test_server module directly will continue to work in OTP 19. The test suites for Erlang/OTP are built and executed in exactly the same way as previously. Here are some more details. The modules test_server*.erl and erl2html2.erl in lib/test_server/src have been moved to common_test/src. The test_server.hrl and test_server_line.hrl include files have been deleted. The macros in test_server.hrl have been copied into lib/common_test/include/ct.hrl. The ts*.erl modules and their associated data files in lib/test_server/src has been been moved to the new directory lib/common_test/test_server. The ts* modules are no longer built to lib/common_test/ebin. They will only built when 'make release_tests' is executed. The test suite for test_server has been moved to lib/common_test/test. The rest of the files have been deleted.
Diffstat (limited to 'lib/common_test/test_server/ts.erl')
-rw-r--r--lib/common_test/test_server/ts.erl1019
1 files changed, 1019 insertions, 0 deletions
diff --git a/lib/common_test/test_server/ts.erl b/lib/common_test/test_server/ts.erl
new file mode 100644
index 0000000000..8bbdc8f8cf
--- /dev/null
+++ b/lib/common_test/test_server/ts.erl
@@ -0,0 +1,1019 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 1997-2014. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File : ts.erl
+%%% Purpose : Frontend for running tests.
+%%%-------------------------------------------------------------------
+
+-module(ts).
+
+-export([cl_run/1,
+ run/0, run/1, run/2, run/3, run/4, run/5,
+ run_category/1, run_category/2, run_category/3,
+ tests/0, tests/1, suites/1, categories/1,
+ install/0, install/1,
+ estone/0, estone/1,
+ cross_cover_analyse/1,
+ compile_testcases/0, compile_testcases/1,
+ help/0]).
+
+%% Functions kept for backwards compatibility
+-export([bench/0, bench/1, bench/2, benchmarks/0,
+ smoke_test/0, smoke_test/1,smoke_test/2, smoke_tests/0]).
+
+-export([i/0, l/1, r/0, r/1, r/2, r/3]).
+
+%%%----------------------------------------------------------------------
+%%% This module, ts, is the interface to all of the functionality of
+%%% the TS framework. The picture below shows the relationship of
+%%% the modules:
+%%%
+%%% +-- ts_install --+------ ts_autoconf_win32
+%%% |
+%%% ts ---+ +------ ts_erl_config
+%%% | | ts_lib
+%%% +-- ts_run -----+------ ts_make
+%%% | | ts_filelib
+%%% | +------ ts_make_erl
+%%% |
+%%% +-- ts_benchmark
+%%%
+%%% The modules ts_lib and ts_filelib contains utilities used by
+%%% the other modules.
+%%%
+%%% Module Description
+%%% ------ -----------
+%%% ts Frontend to the test server framework. Contains all
+%%% interface functions.
+%%% ts_install Installs the test suite. On Unix, `autoconf' is
+%%% is used; on Windows, ts_autoconf_win32 is used.
+%%% The result is written to the file `variables'.
+%%% ts_run Supervises running of the tests.
+%%% ts_autconf_win32 An `autoconf' for Windows.
+%%% ts_autconf_cross_env `autoconf' for other platforms (cross environment)
+%%% ts_erl_config Finds out information about the Erlang system,
+%%% for instance the location of erl_interface.
+%%% This works for either an installed OTP or an Erlang
+%%% system running in a git repository/source tree.
+%%% ts_make Interface to run the `make' program on Unix
+%%% and other platforms.
+%%% ts_make_erl A corrected version of the standar Erlang module
+%%% make (used for rebuilding test suites).
+%%% ts_lib Miscellanous utility functions, each used by several
+%%% other modules.
+%%% ts_benchmark Supervises otp benchmarks and collects results.
+%%%----------------------------------------------------------------------
+
+-include_lib("kernel/include/file.hrl").
+-include("ts.hrl").
+
+-define(
+ install_help,
+ [
+ " ts:install()\n",
+ " Install ts with no options.\n",
+ "\n",
+ " ts:install(Options)\n",
+ " Install ts with a list of options, see below.\n",
+ "\n",
+ "Installation options supported:\n\n",
+ " {longnames, true} - Use fully qualified hostnames\n",
+ " {verbose, Level} - Sets verbosity level for TS output (0,1,2), 0 is\n"
+ " quiet(default).\n"
+ " {crossroot, ErlTop}\n"
+ " - Erlang root directory on build host, ~n"
+ " normally same value as $ERL_TOP\n"
+ " {crossenv, [{Key,Val}]}\n"
+ " - Environmentals used by test configure on build host\n"
+ " {crossflags, FlagsString}\n"
+ " - Flags used by test configure on build host\n"
+ " {xcomp, XCompFile}\n"
+ " - The xcomp file to use for cross compiling the~n"
+ " testcases. Using this option will override any~n"
+ " cross* configurations given to ts. Note that you~n"
+ " have to have a correct ERL_TOP as well.~n"
+ ]).
+
+help() ->
+ case filelib:is_file(?variables) of
+ false -> help(uninstalled);
+ true -> help(installed)
+ end.
+
+help(uninstalled) ->
+ H = ["ts is not yet installed. To install use:\n\n"],
+ show_help([H,?install_help]);
+help(installed) ->
+ H = ["\n",
+ "Run functions:\n\n",
+ " ts:run()\n",
+ " Run the tests for all apps. The tests are defined by the\n",
+ " main test specification for each app: ../App_test/App.spec.\n",
+ "\n",
+ " ts:run(Apps)\n",
+ " Apps = atom() | [atom()]\n",
+ " Run the tests for an app, or set of apps. The tests are\n",
+ " defined by the main test specification for each app:\n",
+ " ../App_test/App.spec.\n",
+ "\n",
+ " ts:run(App, Suites)\n",
+ " App = atom(), Suites = atom() | [atom()]\n",
+ " Run one or more test suites for App (i.e. modules named\n",
+ " *_SUITE.erl, located in ../App_test/).\n",
+ "\n",
+ " ts:run(App, Suite, TestCases)\n",
+ " App = atom(), Suite = atom(),\n",
+ " TestCases = TCs | {testcase,TCs}, TCs = atom() | [atom()]\n",
+ " Run one or more test cases (functions) in Suite.\n",
+ "\n",
+ " ts:run(App, Suite, {group,Groups})\n",
+ " App = atom(), Suite = atom(), Groups = atom() | [atom()]\n",
+ " Run one or more test case groups in Suite.\n",
+ "\n",
+ " ts:run(App, Suite, {group,Group}, {testcase,TestCases})\n",
+ " App = atom(), Suite = atom(), Group = atom(),\n",
+ " TestCases = atom() | [atom()]\n",
+ " Run one or more test cases in a test case group in Suite.\n",
+ "\n",
+ " ts:run_category(TestCategory)\n",
+ " TestCategory = smoke | essential | bench | atom()\n",
+ " Run the specified category of tests for all apps.\n",
+ " For each app, the tests are defined by the specification:\n",
+ " ../App_test/App_TestCategory.spec.\n",
+ "\n",
+ " ts:run_category(Apps, TestCategory)\n",
+ " Apps = atom() | [atom()],\n",
+ " TestCategory = smoke | essential | bench | atom()\n",
+ " Run the specified category of tests for the given app or apps.\n",
+ "\n",
+ " Note that the test category parameter may have arbitrary value,\n",
+ " but should correspond to an existing test specification with file\n",
+ " name: ../App_test/App_TestCategory.spec.\n",
+ " Predefined categories exist for smoke tests, essential tests and\n",
+ " benchmark tests. The corresponding specs are:\n",
+ " ../*_test/Spec_smoke.spec, ../*_test/Spec_essential.spec and\n",
+ " ../*_test/Spec_bench.spec.\n",
+ "\n",
+ " All above run functions can take an additional last argument,\n",
+ " Options, which is a list of options (e.g. ts:run(App, Options),\n",
+ " or ts:run_category(Apps, TestCategory, Options)).\n",
+ "\n",
+ "Run options supported:\n\n",
+ " batch - Do not start a new xterm\n",
+ " {verbose, Level} - Same as the verbosity option for install\n",
+ " verbose - Same as {verbose, 1}\n",
+ " {vars, Vars} - Variables in addition to the 'variables' file\n",
+ " Can be any of the install options\n",
+ " {trace, TraceSpec}- Start call trace on target and slave nodes\n",
+ " TraceSpec is the name of a file containing\n",
+ " trace specifications or a list of trace\n",
+ " specification elements.\n",
+ " {config, Path} - Specify which directory ts should get it's \n"
+ " config files from. The files should follow\n"
+ " the convention lib/test_server/src/ts*.config.\n"
+ " These config files can also be specified by\n"
+ " setting the TEST_CONFIG_PATH environment\n"
+ " variable to the directory where the config\n"
+ " files are. The default location is\n"
+ " tests/test_server/.\n"
+ "\n",
+ "Supported trace information elements:\n\n",
+ " {tp | tpl, Mod, [] | match_spec()}\n",
+ " {tp | tpl, Mod, Func, [] | match_spec()}\n",
+ " {tp | tpl, Mod, Func, Arity, [] | match_spec()}\n",
+ " {ctp | ctpl, Mod}\n",
+ " {ctp | ctpl, Mod, Func}\n",
+ " {ctp | ctpl, Mod, Func, Arity}\n",
+ "\n\n",
+ "Support functions:\n\n",
+ " ts:tests()\n",
+ " Returns all apps available for testing.\n",
+ "\n",
+ " ts:tests(TestCategory)\n",
+ " Returns all apps that provide tests in the given category.\n",
+ "\n",
+ " ts:suites(App)\n",
+ " Returns all available test suites for App,\n",
+ " i.e. ../App_test/*_SUITE.erl\n",
+ "\n",
+ " ts:categories(App)\n",
+ " Returns all test categories available for App.\n",
+ "\n",
+ " ts:estone()\n",
+ " Runs estone_SUITE in the kernel application with no run options\n",
+ "\n",
+ " ts:estone(Opts)\n",
+ " Runs estone_SUITE in the kernel application with the given\n",
+ " run options\n",
+ "\n",
+ " ts:cross_cover_analyse(Level)\n",
+ " Use after ts:run with option cover or cover_details. Analyses\n",
+ " modules specified with a 'cross' statement in the cover spec file.\n",
+ " Level can be 'overview' or 'details'.\n",
+ "\n",
+ " ts:compile_testcases()\n",
+ " ts:compile_testcases(Apps)\n",
+ " Compiles all test cases for the given apps, for usage in a\n",
+ " cross compilation environment.\n",
+ "\n\n",
+ "Installation (already done):\n\n"
+ ],
+ show_help([H,?install_help]).
+
+show_help(H) ->
+ io:format(lists:flatten(H)).
+
+
+%% Installs tests.
+install() ->
+ ts_install:install(install_local,[]).
+install(Options) when is_list(Options) ->
+ ts_install:install(install_local,Options).
+
+%% run/0
+%% Runs all specs found by ts:tests(), if any, or returns
+%% {error, no_tests_available}. (batch)
+run() ->
+ case ts:tests() of
+ [] ->
+ {error, no_tests_available};
+ _ ->
+ check_and_run(fun(Vars) -> run_all(Vars) end)
+ end.
+run_all(_Vars) ->
+ run_some(tests(), [batch]).
+
+run_some([], _Opts) ->
+ ok;
+run_some(Apps, Opts) ->
+ case proplists:get_value(test_category, Opts) of
+ bench ->
+ check_and_run(fun(Vars) -> ts_benchmark:run(Apps, Opts, Vars) end);
+ _Other ->
+ run_some1(Apps, Opts)
+ end.
+
+run_some1([], _Opts) ->
+ ok;
+run_some1([{App,Mod}|Apps], Opts) ->
+ case run(App, Mod, Opts) of
+ ok -> ok;
+ Error -> io:format("~p: ~p~n",[{App,Mod},Error])
+ end,
+ run_some1(Apps, Opts);
+run_some1([App|Apps], Opts) ->
+ case run(App, Opts) of
+ ok -> ok;
+ Error -> io:format("~p: ~p~n",[App,Error])
+ end,
+ run_some1(Apps, Opts).
+
+%% This can be used from command line. Both App and
+%% TestCategory must be specified. App may be 'all'
+%% and TestCategory may be 'main'. Examples:
+%% erl -s ts cl_run kernel smoke <options>
+%% erl -s ts cl_run kernel main <options>
+%% erl -s ts cl_run all essential <options>
+%% erl -s ts cl_run all main <options>
+%% When using the 'main' category and running with cover,
+%% one can also use the cross_cover_analysis flag.
+cl_run([App,Cat|Options0]) when is_atom(App) ->
+
+ AllAtomsFun = fun(X) when is_atom(X) -> true;
+ (_) -> false
+ end,
+ Options1 =
+ case lists:all(AllAtomsFun, Options0) of
+ true ->
+ %% Could be from command line
+ lists:map(fun(Opt) ->
+ to_erlang_term(Opt)
+ end, Options0) -- [batch];
+ false ->
+ Options0 -- [batch]
+ end,
+ %% Make sure there is exactly one occurence of 'batch'
+ Options2 = [batch|Options1],
+
+ Result =
+ case {App,Cat} of
+ {all,main} ->
+ run(tests(), Options2);
+ {all,Cat} ->
+ run_category(Cat, Options2);
+ {_,main} ->
+ run(App, Options2);
+ {_,Cat} ->
+ run_category(App, Cat, Options2)
+ end,
+ case check_for_cross_cover_analysis_flag(Options2) of
+ false ->
+ ok;
+ Level ->
+ cross_cover_analyse(Level)
+ end,
+ Result.
+
+%% run/1
+%% Runs tests for one app (interactive).
+run(App) when is_atom(App) ->
+ Options = check_test_get_opts(App, []),
+ File = atom_to_list(App),
+ run_test(File, [{spec,[File++".spec"]},{allow_user_terms,true}], Options);
+
+%% This can be used from command line, e.g.
+%% erl -s ts run all <options>
+%% erl -s ts run main <options>
+run([all,main|Opts]) ->
+ cl_run([all,main|Opts]);
+run([all|Opts]) ->
+ cl_run([all,main|Opts]);
+run([main|Opts]) ->
+ cl_run([all,main|Opts]);
+%% Backwards compatible
+run([all_tests|Opts]) ->
+ cl_run([all,main|Opts]);
+
+%% run/1
+%% Runs the main tests for all available apps
+run(Apps) when is_list(Apps) ->
+ run(Apps, [batch]).
+
+%% run/2
+%% Runs the main tests for all available apps
+run(Apps, Opts) when is_list(Apps), is_list(Opts) ->
+ run_some(Apps, Opts);
+
+%% Runs tests for one app with list of suites or with options
+run(App, ModsOrOpts) when is_atom(App),
+ is_list(ModsOrOpts) ->
+ case is_list_of_suites(ModsOrOpts) of
+ false ->
+ run(App, {opts_list,ModsOrOpts});
+ true ->
+ run_some([{App,M} || M <- ModsOrOpts],
+ [batch])
+ end;
+
+run(App, {opts_list,Opts}) ->
+ Options = check_test_get_opts(App, Opts),
+ File = atom_to_list(App),
+
+ %% check if other test category than main has been specified
+ {CatSpecName,TestCat} =
+ case proplists:get_value(test_category, Opts) of
+ undefined ->
+ {"",main};
+ Cat ->
+ {"_" ++ atom_to_list(Cat),Cat}
+ end,
+
+ WhatToDo =
+ case App of
+ %% Known to exist but fails generic tests below
+ emulator -> test;
+ system -> test;
+ erl_interface -> test;
+ epmd -> test;
+ _ ->
+ case code:lib_dir(App) of
+ {error,bad_name} ->
+ %% Application does not exist
+ skip;
+ Path ->
+ case file:read_file_info(filename:join(Path,"ebin")) of
+ {ok,#file_info{type=directory}} ->
+ %% Erlang application is built
+ test;
+ _ ->
+ case filelib:wildcard(
+ filename:join([Path,"priv","*.jar"])) of
+ [] ->
+ %% The application is not built
+ skip;
+ [_|_] ->
+ %% Java application is built
+ test
+ end
+ end
+ end
+ end,
+ case WhatToDo of
+ skip ->
+ SkipSpec = create_skip_spec(App, suites(App)),
+ run_test(File, [{spec,[SkipSpec]}], Options);
+ test when TestCat == bench ->
+ check_and_run(fun(Vars) ->
+ ts_benchmark:run([App], Options, Vars)
+ end);
+ test ->
+ Spec = File ++ CatSpecName ++ ".spec",
+ run_test(File, [{spec,[Spec]},{allow_user_terms,true}], Options)
+ end;
+
+%% Runs one module for an app (interactive)
+run(App, Mod) when is_atom(App), is_atom(Mod) ->
+ run_test({atom_to_list(App),Mod},
+ [{suite,Mod}],
+ [interactive]).
+
+%% run/3
+%% Run one module for an app with Opts
+run(App, Mod, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
+ run_test({atom_to_list(App),Mod},
+ [{suite,Mod}], Options);
+
+%% Run multiple modules with Opts
+run(App, Mods, Opts) when is_atom(App),
+ is_list(Mods),
+ is_list(Opts) ->
+ run_some([{App,M} || M <- Mods], Opts);
+
+%% Runs one test case in a module.
+run(App, Mod, Case) when is_atom(App),
+ is_atom(Mod),
+ is_atom(Case) ->
+ Options = check_test_get_opts(App, []),
+ Args = [{suite,Mod},{testcase,Case}],
+ run_test(atom_to_list(App), Args, Options);
+
+%% Runs one or more groups in a module.
+run(App, Mod, Grs={group,_Groups}) when is_atom(App),
+ is_atom(Mod) ->
+ Options = check_test_get_opts(App, []),
+ Args = [{suite,Mod},Grs],
+ run_test(atom_to_list(App), Args, Options);
+
+%% Runs one or more test cases in a module.
+run(App, Mod, TCs={testcase,_Cases}) when is_atom(App),
+ is_atom(Mod) ->
+ Options = check_test_get_opts(App, []),
+ Args = [{suite,Mod},TCs],
+ run_test(atom_to_list(App), Args, Options).
+
+%% run/4
+%% Run one test case in a module with Options.
+run(App, Mod, Case, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_atom(Case),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
+ Args = [{suite,Mod},{testcase,Case}],
+ run_test(atom_to_list(App), Args, Options);
+
+%% Run one or more test cases in a module with Options.
+run(App, Mod, {testcase,Cases}, Opts) when is_atom(App),
+ is_atom(Mod) ->
+ run(App, Mod, Cases, Opts);
+run(App, Mod, Cases, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Cases),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
+ Args = [{suite,Mod},Cases],
+ run_test(atom_to_list(App), Args, Options);
+
+%% Run one or more test cases in a group.
+run(App, Mod, Gr={group,_Group}, {testcase,Cases}) when is_atom(App),
+ is_atom(Mod) ->
+ run(App, Mod, Gr, Cases, [batch]);
+
+
+%% Run one or more groups in a module with Options.
+run(App, Mod, Grs={group,_Groups}, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Opts) ->
+ Options = check_test_get_opts(App, Opts),
+ Args = [{suite,Mod},Grs],
+ run_test(atom_to_list(App), Args, Options).
+
+%% run/5
+%% Run one or more test cases in a group with Options.
+run(App, Mod, Group, Cases, Opts) when is_atom(App),
+ is_atom(Mod),
+ is_list(Opts) ->
+ Group1 = if is_tuple(Group) -> Group; true -> {group,Group} end,
+ Cases1 = if is_tuple(Cases) -> Cases; true -> {testcase,Cases} end,
+ Options = check_test_get_opts(App, Opts),
+ Args = [{suite,Mod},Group1,Cases1],
+ run_test(atom_to_list(App), Args, Options).
+
+%% run_category/1
+run_category(TestCategory) when is_atom(TestCategory) ->
+ run_category(TestCategory, [batch]).
+
+%% run_category/2
+run_category(TestCategory, Opts) when is_atom(TestCategory),
+ is_list(Opts) ->
+ case ts:tests(TestCategory) of
+ [] ->
+ {error, no_tests_available};
+ Apps ->
+ Opts1 = [{test_category,TestCategory} | Opts],
+ run_some(Apps, Opts1)
+ end;
+
+run_category(Apps, TestCategory) when is_atom(TestCategory) ->
+ run_category(Apps, TestCategory, [batch]).
+
+%% run_category/3
+run_category(App, TestCategory, Opts) ->
+ Apps = if is_atom(App) -> [App];
+ is_list(App) -> App
+ end,
+ Opts1 = [{test_category,TestCategory} | Opts],
+ run_some(Apps, Opts1).
+
+%%-----------------------------------------------------------------
+%% Functions kept for backwards compatibility
+
+bench() ->
+ run_category(bench, []).
+bench(Opts) when is_list(Opts) ->
+ run_category(bench, Opts);
+bench(App) ->
+ run_category(App, bench, []).
+bench(App, Opts) when is_atom(App) ->
+ run_category(App, bench, Opts);
+bench(Apps, Opts) when is_list(Apps) ->
+ run_category(Apps, bench, Opts).
+
+benchmarks() ->
+ tests(bench).
+
+smoke_test() ->
+ run_category(smoke, []).
+smoke_test(Opts) when is_list(Opts) ->
+ run_category(smoke, Opts);
+smoke_test(App) ->
+ run_category(App, smoke, []).
+smoke_test(App, Opts) when is_atom(App) ->
+ run_category(App, smoke, Opts);
+smoke_test(Apps, Opts) when is_list(Apps) ->
+ run_category(Apps, smoke, Opts).
+
+smoke_tests() ->
+ tests(smoke).
+
+%%-----------------------------------------------------------------
+
+is_list_of_suites(List) ->
+ lists:all(fun(Suite) ->
+ S = if is_atom(Suite) -> atom_to_list(Suite);
+ true -> Suite
+ end,
+ try lists:last(string:tokens(S,"_")) of
+ "SUITE" -> true;
+ "suite" -> true;
+ _ -> false
+ catch
+ _:_ -> false
+ end
+ end, List).
+
+%% Create a spec to skip all SUITES, this is used when the application
+%% to be tested is not part of the OTP release to be tested.
+create_skip_spec(App, SuitesToSkip) ->
+ {ok,Cwd} = file:get_cwd(),
+ AppString = atom_to_list(App),
+ Specname = AppString++"_skip.spec",
+ {ok,D} = file:open(filename:join([filename:dirname(Cwd),
+ AppString++"_test",Specname]),
+ [write]),
+ TestDir = "\"../"++AppString++"_test\"",
+ io:format(D,"{suites, "++TestDir++", all}.~n",[]),
+ io:format(D,"{skip_suites, "++TestDir++", ~w, \"Skipped as application"
+ " is not in path!\"}.",[SuitesToSkip]),
+ Specname.
+
+%% Check testspec for App to be valid and get possible options
+%% from the list.
+check_test_get_opts(App, Opts) ->
+ validate_test(App),
+ Mode = configmember(batch, {batch, interactive}, Opts),
+ Vars = configvars(Opts),
+ Trace = get_config(trace,Opts),
+ ConfigPath = get_config(config,Opts),
+ KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Opts),
+ Cover = configcover(App,Opts),
+ lists:flatten([Vars,Mode,Trace,KeepTopcase,Cover,ConfigPath]).
+
+to_erlang_term(Atom) ->
+ String = atom_to_list(Atom),
+ {ok, Tokens, _} = erl_scan:string(lists:append([String, ". "])),
+ {ok, Term} = erl_parse:parse_term(Tokens),
+ Term.
+
+%% Validate that Testspec really is a testspec,
+%% and exit if not.
+validate_test(Testspec) ->
+ case lists:member(Testspec, tests()) of
+ true ->
+ ok;
+ false ->
+ io:format("This testspec does not seem to be "
+ "available.~n Please try ts:tests() "
+ "to see available tests.~n"),
+ exit(self(), {error, test_not_available})
+ end.
+
+configvars(Opts) ->
+ case lists:keysearch(vars, 1, Opts) of
+ {value, {vars, List}} ->
+ List0 = special_vars(Opts),
+ Key = fun(T) -> element(1,T) end,
+ DelDupList =
+ lists:filter(fun(V) ->
+ case lists:keysearch(Key(V),1,List0) of
+ {value,_} -> false;
+ _ -> true
+ end
+ end, List),
+ {vars, [List0|DelDupList]};
+ _ ->
+ {vars, special_vars(Opts)}
+ end.
+
+%% Allow some shortcuts in the options...
+special_vars(Opts) ->
+ SpecVars =
+ case lists:member(verbose, Opts) of
+ true ->
+ [{verbose, 1}];
+ false ->
+ case lists:keysearch(verbose, 1, Opts) of
+ {value, {verbose, Lvl}} ->
+ [{verbose, Lvl}];
+ _ ->
+ [{verbose, 0}]
+ end
+ end,
+ SpecVars1 =
+ case lists:keysearch(diskless, 1, Opts) of
+ {value,{diskless, true}} ->
+ [{diskless, true} | SpecVars];
+ _ ->
+ SpecVars
+ end,
+ case lists:keysearch(testcase_callback, 1, Opts) of
+ {value,{testcase_callback, CBM, CBF}} ->
+ [{ts_testcase_callback, {CBM,CBF}} | SpecVars1];
+ {value,{testcase_callback, CB}} ->
+ [{ts_testcase_callback, CB} | SpecVars1];
+ _ ->
+ SpecVars1
+ end.
+
+get_config(Key,Config) ->
+ case lists:keysearch(Key,1,Config) of
+ {value,Value} -> Value;
+ false -> []
+ end.
+
+configcover(Testspec,[cover|_]) ->
+ {cover,Testspec,default_coverfile(Testspec),overview};
+configcover(Testspec,[cover_details|_]) ->
+ {cover,Testspec,default_coverfile(Testspec),details};
+configcover(Testspec,[{cover,File}|_]) ->
+ {cover,Testspec,File,overview};
+configcover(Testspec,[{cover_details,File}|_]) ->
+ {cover,Testspec,File,details};
+configcover(Testspec,[_H|T]) ->
+ configcover(Testspec,T);
+configcover(_Testspec,[]) ->
+ [].
+
+default_coverfile(Testspec) ->
+ {ok,Cwd} = file:get_cwd(),
+ CoverFile = filename:join([filename:dirname(Cwd),
+ atom_to_list(Testspec)++"_test",
+ atom_to_list(Testspec)++".cover"]),
+ case filelib:is_file(CoverFile) of
+ true ->
+ CoverFile;
+ false ->
+ none
+ end.
+
+configmember(Member, {True, False}, Config) ->
+ case lists:member(Member, Config) of
+ true ->
+ True;
+ false ->
+ False
+ end.
+
+
+check_for_cross_cover_analysis_flag(Config) ->
+ check_for_cross_cover_analysis_flag(Config,false,false).
+check_for_cross_cover_analysis_flag([cover|Config],false,false) ->
+ check_for_cross_cover_analysis_flag(Config,overview,false);
+check_for_cross_cover_analysis_flag([cover|_Config],false,true) ->
+ overview;
+check_for_cross_cover_analysis_flag([cover_details|Config],false,false) ->
+ check_for_cross_cover_analysis_flag(Config,details,false);
+check_for_cross_cover_analysis_flag([cover_details|_Config],false,true) ->
+ details;
+check_for_cross_cover_analysis_flag([cross_cover_analysis|Config],false,_) ->
+ check_for_cross_cover_analysis_flag(Config,false,true);
+check_for_cross_cover_analysis_flag([cross_cover_analysis|_Config],Level,_) ->
+ Level;
+check_for_cross_cover_analysis_flag([_|Config],Level,CrossFlag) ->
+ check_for_cross_cover_analysis_flag(Config,Level,CrossFlag);
+check_for_cross_cover_analysis_flag([],_,_) ->
+ false.
+
+
+%% Returns all available apps.
+tests() ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:specs(Cwd).
+
+%% Returns all apps that provide tests in the given test category
+tests(main) ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:specs(Cwd);
+tests(bench) ->
+ ts_benchmark:benchmarks();
+tests(TestCategory) ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:specialized_specs(Cwd, atom_to_list(TestCategory)).
+
+%% Returns a list of available test suites for App.
+suites(App) ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:suites(Cwd, atom_to_list(App)).
+
+%% Returns all available test categories for App
+categories(App) ->
+ {ok, Cwd} = file:get_cwd(),
+ ts_lib:test_categories(Cwd, atom_to_list(App)).
+
+%%
+%% estone/0, estone/1
+%% Opts = same as Opts or Config for the run(...) function,
+%% e.g. [batch]
+%%
+estone() -> run(emulator,estone_SUITE).
+estone(Opts) when is_list(Opts) -> run(emulator,estone_SUITE,Opts).
+
+%%
+%% cross_cover_analyse/1
+%% Level = details | overview
+%% Can be called on any node after a test (with cover) is
+%% completed. The node's current directory must be the same as when
+%% the tests were run.
+%%
+cross_cover_analyse([Level]) ->
+ cross_cover_analyse(Level);
+cross_cover_analyse(Level) ->
+ Apps = get_last_app_tests(),
+ test_server_ctrl:cross_cover_analyse(Level,Apps).
+
+get_last_app_tests() ->
+ AllTests = filelib:wildcard(filename:join(["*","*_test.logs"])),
+ {ok,RE} = re:compile("^[^/]*/[^\.]*\.(.*)_test\.logs$"),
+ get_last_app_tests(AllTests,RE,[]).
+
+get_last_app_tests([Dir|Dirs],RE,Acc) ->
+ NewAcc =
+ case re:run(Dir,RE,[{capture,all,list}]) of
+ {match,[Dir,AppStr]} ->
+ Dir1 = filename:dirname(Dir), % cover logs in ct_run.<t> dir
+ App = list_to_atom(AppStr),
+ case lists:keytake(App,1,Acc) of
+ {value,{App,LastDir},Rest} ->
+ if Dir1 > LastDir ->
+ [{App,Dir1}|Rest];
+ true ->
+ Acc
+ end;
+ false ->
+ [{App,Dir1} | Acc]
+ end;
+ _ ->
+ Acc
+ end,
+ get_last_app_tests(Dirs,RE,NewAcc);
+get_last_app_tests([],_,Acc) ->
+ Acc.
+
+%%% Implementation.
+
+check_and_run(Fun) ->
+ case file:consult(?variables) of
+ {ok, Vars} ->
+ check_and_run(Fun, Vars);
+ {error, Error} when is_atom(Error) ->
+ {error, not_installed};
+ {error, Reason} ->
+ {error, {bad_installation, file:format_error(Reason)}}
+ end.
+
+check_and_run(Fun, Vars) ->
+ Platform = ts_install:platform_id(Vars),
+ case lists:keysearch(platform_id, 1, Vars) of
+ {value, {_, Platform}} ->
+ case catch apply(Fun, [Vars]) of
+ {'EXIT', Reason} ->
+ exit(Reason);
+ Other ->
+ Other
+ end;
+ {value, {_, OriginalPlatform}} ->
+ io:format("These test suites were installed for '~s'.\n",
+ [OriginalPlatform]),
+ io:format("But the current platform is '~s'.\nPlease "
+ "install for this platform before running "
+ "any tests.\n", [Platform]),
+ {error, inconsistent_platforms};
+ false ->
+ {error, {bad_installation, no_platform}}
+ end.
+
+run_test(File, Args, Options) ->
+ check_and_run(fun(Vars) -> run_test(File, Args, Options, Vars) end).
+
+run_test(File, Args, Options, Vars) ->
+ ts_run:run(File, Args, Options, Vars).
+
+
+%% This module provides some convenient shortcuts to running
+%% the test server from within a started Erlang shell.
+%% (This are here for backwards compatibility.)
+%%
+%% r()
+%% r(Opts)
+%% r(SpecOrMod)
+%% r(SpecOrMod, Opts)
+%% r(Mod, Case)
+%% r(Mod, Case, Opts)
+%% Each of these functions starts the test server if it
+%% isn't already running, then runs the test case(s) selected
+%% by the aguments.
+%% SpecOrMod can be a module name or the name of a test spec file,
+%% with the extension .spec or .spec.OsType. The module Mod will
+%% be reloaded before running the test cases.
+%% Opts = [Opt],
+%% Opt = {Cover,AppOrCoverFile} | {Cover,App,CoverFile}
+%% Cover = cover | cover_details
+%% AppOrCoverFile = App | CoverFile
+%% App = atom(), an application name
+%% CoverFile = string(), name of a cover file
+%% (see doc of test_server_ctrl:cover/2/3)
+%%
+%% i()
+%% Shows information about the jobs being run, by dumping
+%% the process information for the test_server.
+%%
+%% l(Mod)
+%% This function reloads a module just like c:l/1, but works
+%% even for a module in one of the sticky library directories
+%% (for instance, lists can be reloaded).
+
+%% Runs all tests cases in the current directory.
+
+r() ->
+ r([]).
+r(Opts) when is_list(Opts), is_atom(hd(Opts)) ->
+ ensure_ts_started(Opts),
+ test_server_ctrl:add_dir("current_dir", ".");
+
+%% Checks if argument is a spec file or a module
+%% (spec file must be named "*.spec" or "*.spec.OsType")
+%% If module, reloads module and runs all test cases in it.
+%% If spec, runs all test cases in it.
+
+r(SpecOrMod) ->
+ r(SpecOrMod,[]).
+r(SpecOrMod,Opts) when is_list(Opts) ->
+ ensure_ts_started(Opts),
+ case filename:extension(SpecOrMod) of
+ [] ->
+ l(SpecOrMod),
+ test_server_ctrl:add_module(SpecOrMod);
+ ".spec" ->
+ test_server_ctrl:add_spec(SpecOrMod);
+ _ ->
+ Spec2 = filename:rootname(SpecOrMod),
+ case filename:extension(Spec2) of
+ ".spec" ->
+ %% *.spec.Type
+ test_server_ctrl:add_spec(SpecOrMod);
+ _ ->
+ {error, unknown_filetype}
+ end
+ end;
+
+%% Reloads the given module and runs the given test case in it.
+
+r(Mod, Case) ->
+ r(Mod,Case,[]).
+r(Mod, Case, Opts) ->
+ ensure_ts_started(Opts),
+ l(Mod),
+ test_server_ctrl:add_case(Mod, Case).
+
+%% Shows information about the jobs being run.
+
+i() ->
+ ensure_ts_started([]),
+ hformat("Job", "Current", "Total", "Success", "Failed", "Skipped"),
+ i(test_server_ctrl:jobs()).
+
+i([{Name, Pid}|Rest]) when is_pid(Pid) ->
+ {dictionary, PI} = process_info(Pid, dictionary),
+ {value, {_, CaseNum}} = lists:keysearch(test_server_case_num, 1, PI),
+ {value, {_, Cases}} = lists:keysearch(test_server_cases, 1, PI),
+ {value, {_, Failed}} = lists:keysearch(test_server_failed, 1, PI),
+ {value, {_, {UserSkipped,AutoSkipped}}} = lists:keysearch(test_server_skipped, 1, PI),
+ {value, {_, Ok}} = lists:keysearch(test_server_ok, 1, PI),
+ nformat(Name, CaseNum, Cases, Ok, Failed, UserSkipped+AutoSkipped),
+ i(Rest);
+i([]) ->
+ ok.
+
+hformat(A1, A2, A3, A4, A5, A6) ->
+ io:format("~-20s ~8s ~8s ~8s ~8s ~8s~n", [A1,A2,A3,A4,A5,A6]).
+
+nformat(A1, A2, A3, A4, A5, A6) ->
+ io:format("~-20s ~8w ~8w ~8w ~8w ~8w~n", [A1,A2,A3,A4,A5,A6]).
+
+%% Force load of a module even if it is in a sticky directory.
+
+l(Mod) ->
+ case do_load(Mod) of
+ {error, sticky_directory} ->
+ Dir = filename:dirname(code:which(Mod)),
+ code:unstick_dir(Dir),
+ do_load(Mod),
+ code:stick_dir(Dir);
+ X ->
+ X
+ end.
+
+
+ensure_ts_started(Opts) ->
+ Pid = case whereis(test_server_ctrl) of
+ undefined ->
+ test_server_ctrl:start();
+ P when is_pid(P) ->
+ P
+ end,
+ case Opts of
+ [{Cover,AppOrCoverFile}] when Cover==cover; Cover==cover_details ->
+ test_server_ctrl:cover(AppOrCoverFile,cover_type(Cover));
+ [{Cover,App,CoverFile}] when Cover==cover; Cover==cover_details ->
+ test_server_ctrl:cover(App,CoverFile,cover_type(Cover));
+ _ ->
+ ok
+ end,
+ Pid.
+
+cover_type(cover) -> overview;
+cover_type(cover_details) -> details.
+
+do_load(Mod) ->
+ code:purge(Mod),
+ code:load_file(Mod).
+
+
+compile_testcases() ->
+ compile_datadirs("../*/*_data").
+
+compile_testcases(App) when is_atom(App) ->
+ compile_testcases([App]);
+compile_testcases([App | T]) ->
+ compile_datadirs(io_lib:format("../~s_test/*_data", [App])),
+ compile_testcases(T);
+compile_testcases([]) ->
+ ok.
+
+compile_datadirs(DataDirs) ->
+ {ok,Variables} = file:consult("variables"),
+
+ lists:foreach(fun(Dir) ->
+ ts_lib:make_non_erlang(Dir, Variables)
+ end,
+ filelib:wildcard(DataDirs)).