aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2012-08-15 16:48:32 +0200
committerPeter Andersson <[email protected]>2012-08-21 11:50:20 +0200
commitecf119feda32c0692bf688462ced24efa35789f1 (patch)
tree69b83cd13f0d847e6b00c4d5231d39d1d4f9eb26
parent306bcdfd962b941434c37d9059ee4dc0364cc71a (diff)
downloadotp-ecf119feda32c0692bf688462ced24efa35789f1.tar.gz
otp-ecf119feda32c0692bf688462ced24efa35789f1.tar.bz2
otp-ecf119feda32c0692bf688462ced24efa35789f1.zip
Test new testspec functionality and correct bugs
OTP-10049 OTP-10089
-rw-r--r--lib/common_test/doc/src/run_test_chapter.xml8
-rw-r--r--lib/common_test/src/ct_run.erl2
-rw-r--r--lib/common_test/src/ct_testspec.erl458
-rw-r--r--lib/common_test/test/Makefile5
-rw-r--r--lib/common_test/test/ct_auto_compile_SUITE.erl187
-rw-r--r--lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl23
-rw-r--r--lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl130
-rw-r--r--lib/common_test/test/ct_basic_html_SUITE.erl180
-rw-r--r--lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl130
-rw-r--r--lib/common_test/test/ct_testspec_2_SUITE.erl751
10 files changed, 1691 insertions, 183 deletions
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 30486d3eec..bc0af63790 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -425,7 +425,7 @@
<p>Below is the test specification syntax. Test specifications can
be used to run tests both in a single test host environment and
in a distributed Common Test environment (Large Scale
- Testing). The node parameters in the init term are only
+ Testing). The node parameters in the <c>init</c> term are only
relevant in the latter (see the
<seealso marker="ct_master_chapter#test_specifications">Large
Scale Testing</seealso> chapter for information). For details on
@@ -575,9 +575,9 @@
<item>Lastly, all suites for systems t3 are to be completely skipped and this
should be explicitly noted in the log files.</item>
</list>
- <p>It is possible to specify initialization options for nodes defined in the
- test specification. Currently, there are options to start the node and/or to
- evaluate any function on the node.
+ <p>With the <c>init</c> term it's possible to specify initialization options
+ for nodes defined in the test specification. Currently, there are options
+ to start the node and/or to evaluate any function on the node.
See the <seealso marker="ct_master_chapter#ct_slave">Automatic startup of
the test target nodes</seealso> chapter for details.</p>
<p>It is possible for the user to provide a test specification that
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index e801ff925a..04bf86c71a 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -2585,7 +2585,7 @@ opts2args(EnvStartOpts) ->
({decrypt,{file,File}}) ->
[{ct_decrypt_file,[File]}];
({basic_html,true}) ->
- ({basic_html,[]});
+ [{basic_html,[]}];
({basic_html,false}) ->
[];
({event_handler,EH}) when is_atom(EH) ->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 293ff00b07..cbd7dc1f35 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -48,16 +48,10 @@
%%% Version 1 - extract and return all tests and skips for Node
%%% (incl all_nodes)
%%%-------------------------------------------------------------------
-prepare_tests(TestSpec,Node) when is_record(TestSpec,testspec) ->
- Node1 = case Node of
- {N,H} when is_atom(N), is_atom(H) ->
- list_to_atom(atom_to_list(N) ++ "@"
- ++ atom_to_list(H));
- N when is_atom(N) ->
- N
- end,
- case lists:keysearch(Node1,1,prepare_tests(TestSpec)) of
- {value,{Node1,Run,Skip}} ->
+prepare_tests(TestSpec,Node) when is_record(TestSpec,testspec),
+ is_atom(Node) ->
+ case lists:keysearch(Node,1,prepare_tests(TestSpec)) of
+ {value,{Node,Run,Skip}} ->
{Run,Skip};
false ->
{[],[]}
@@ -258,22 +252,23 @@ collect_tests_from_file1([Spec|Specs],TestSpec,Relaxed) ->
SpecDir = filename:dirname(filename:absname(Spec)),
case file:consult(Spec) of
{ok,Terms} ->
- TestSpec1 = collect_tests(Terms,
- TestSpec#testspec{spec_dir=SpecDir},
- Relaxed),
- collect_tests_from_file1(Specs,TestSpec1,Relaxed);
+ case collect_tests(Terms,
+ TestSpec#testspec{spec_dir=SpecDir},
+ Relaxed) of
+ TS = #testspec{tests=Tests, logdir=LogDirs} when Specs == [] ->
+ LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
+ TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1};
+ TS = #testspec{alias = As, nodes = Ns} ->
+ TS1 = TS#testspec{alias = lists:reverse(As),
+ nodes = lists:reverse(Ns)},
+ collect_tests_from_file1(Specs,TS1,Relaxed)
+ end;
{error,Reason} ->
ReasonStr =
lists:flatten(io_lib:format("~s",
[file:format_error(Reason)])),
throw({error,{Spec,ReasonStr}})
- end;
-collect_tests_from_file1([],TS=#testspec{config=Cfgs,event_handler=EvHs,
- include=Incl,tests=Tests},_) ->
- TS#testspec{config=lists:reverse(Cfgs),
- event_handler=lists:reverse(EvHs),
- include=lists:reverse(Incl),
- tests=lists:flatten(Tests)}.
+ end.
collect_tests_from_list(Terms,Relaxed) ->
collect_tests_from_list(Terms,[node()],Relaxed).
@@ -287,22 +282,14 @@ collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) ->
E = {error,_} ->
E;
TS ->
- #testspec{config=Cfgs,event_handler=EvHs,
- include=Incl,tests=Tests} = TS,
- TS#testspec{config=lists:reverse(Cfgs),
- event_handler=lists:reverse(EvHs),
- include=lists:reverse(Incl),
- tests=lists:flatten(Tests)}
+ #testspec{tests=Tests, logdir=LogDirs} = TS,
+ LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
+ TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1}
end.
collect_tests(Terms,TestSpec,Relaxed) ->
put(relaxed,Relaxed),
Terms1 = replace_names(Terms),
-
- %%! --- Fri Aug 10 22:51:23 2012 --- peppe was here!
- io:format(user, "~nTERMS:~n", []),
- [io:format(user, "~p~n", [T]) || T <- Terms1],
-
TestSpec1 = get_global(Terms1,TestSpec),
TestSpec2 = get_all_nodes(Terms1,TestSpec1),
{Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2),
@@ -311,23 +298,49 @@ collect_tests(Terms,TestSpec,Relaxed) ->
%% replace names (atoms) in the testspec matching those in 'define' terms by
%% searching recursively through tuples and lists
replace_names(Terms) ->
- Defs = lists:flatmap(fun({define,Name,Replacement}) ->
- [{Name,Replacement}];
- (_) -> []
- end, Terms),
- replace_names(Terms,[],Defs).
-
-replace_names([{define,_,_}|Ts],Modified,Defs) ->
- replace_names(Ts,Modified,Defs);
+ Defs =
+ lists:flatmap(fun(Def={define,Name,_Replacement}) ->
+ %% check that name follows convention
+ if not is_atom(Name) ->
+ throw({illegal_name_in_testspec,Name});
+ true ->
+ [First|_] = atom_to_list(Name),
+ if ((First == $?) or (First == $$)
+ or (First == $_)
+ or ((First >= $A)
+ and (First =< $Z))) ->
+ [Def];
+ true ->
+ throw({illegal_name_in_testspec,
+ Name})
+ end
+ end;
+ (_) -> []
+ end, Terms),
+ DefProps = replace_names_in_defs(Defs,[]),
+ replace_names(Terms,[],DefProps).
+
+replace_names_in_defs([Def|Left],ModDefs) ->
+ [{define,Name,Replacement}] = replace_names([Def],[],ModDefs),
+ replace_names_in_defs(Left,[{Name,Replacement}|ModDefs]);
+replace_names_in_defs([],ModDefs) ->
+ ModDefs.
+
replace_names([Term|Ts],Modified,Defs) when is_tuple(Term) ->
[TypeTag|Data] = tuple_to_list(Term),
Term1 = list_to_tuple([TypeTag|replace_names_in_elems(Data,[],Defs)]),
replace_names(Ts,[Term1|Modified],Defs);
replace_names([Term|Ts],Modified,Defs) when is_atom(Term) ->
case proplists:get_value(Term,Defs) of
- undefined -> replace_names(Ts,[Term|Modified],Defs);
- Replacement -> replace_names(Ts,[Replacement|Modified],Defs)
+ undefined ->
+ replace_names(Ts,[Term|Modified],Defs);
+ Replacement ->
+ replace_names(Ts,[Replacement|Modified],Defs)
end;
+replace_names([Term=[Ch|_]|Ts],Modified,Defs) when is_integer(Ch) ->
+ %% Term *could* be a string, attempt to search through it
+ Term1 = replace_names_in_string(Term,Defs),
+ replace_names(Ts,[Term1|Modified],Defs);
replace_names([Term|Ts],Modified,Defs) ->
replace_names(Ts,[Term|Modified],Defs);
replace_names([],Modified,_Defs) ->
@@ -338,11 +351,22 @@ replace_names_in_elems([Elem|Es],Modified,Defs) when is_tuple(Elem) ->
replace_names_in_elems(Es,[Elem1|Modified],Defs);
replace_names_in_elems([Elem|Es],Modified,Defs) when is_atom(Elem) ->
case proplists:get_value(Elem,Defs) of
- undefined -> replace_names_in_elems(Es,[Elem|Modified],Defs);
- Replacement -> replace_names_in_elems(Es,[Replacement|Modified],Defs)
+ undefined ->
+ %% if Term is a node name, check it for replacements as well
+ Elem1 = replace_names_in_node(Elem,Defs),
+ replace_names_in_elems(Es,[Elem1|Modified],Defs);
+ Replacement ->
+ replace_names_in_elems(Es,[Replacement|Modified],Defs)
+ end;
+replace_names_in_elems([Elem=[Ch|_]|Es],Modified,Defs) when is_integer(Ch) ->
+ %% Term *could* be a string, attempt to search through it
+ case replace_names_in_string(Elem,Defs) of
+ Elem ->
+ List = replace_names_in_elems(Elem,[],Defs),
+ replace_names_in_elems(Es,[List|Modified],Defs);
+ Elem1 ->
+ replace_names_in_elems(Es,[Elem1|Modified],Defs)
end;
-replace_names_in_elems([Str=[Char|_]|Es],Modified,Defs) when is_integer(Char) ->
- replace_names_in_elems(Es,[Str|Modified],Defs);
replace_names_in_elems([Elem|Es],Modified,Defs) when is_list(Elem) ->
List = replace_names_in_elems(Elem,[],Defs),
replace_names_in_elems(Es,[List|Modified],Defs);
@@ -351,6 +375,51 @@ replace_names_in_elems([Elem|Es],Modified,Defs) ->
replace_names_in_elems([],Modified,_Defs) ->
lists:reverse(Modified).
+replace_names_in_string(Term,Defs=[{Name,Replacement=[Ch|_]}|Ds])
+ when is_integer(Ch) ->
+ try re:replace(Term,[$'|atom_to_list(Name)]++"'",
+ Replacement,[{return,list}]) of
+ Term -> % no match, proceed
+ replace_names_in_string(Term,Ds);
+ Term1 ->
+ replace_names_in_string(Term1,Defs)
+ catch
+ _:_ -> Term % Term is not a string
+ end;
+replace_names_in_string(Term,[_|Ds]) ->
+ replace_names_in_string(Term,Ds);
+replace_names_in_string(Term,[]) ->
+ Term.
+
+replace_names_in_node(Node,Defs) ->
+ String = atom_to_list(Node),
+ case lists:member($@,String) of
+ true ->
+ list_to_atom(replace_names_in_node1(String,Defs));
+ false ->
+ Node
+ end.
+
+replace_names_in_node1(NodeStr,Defs=[{Name,Replacement}|Ds]) ->
+ ReplStr = case Replacement of
+ [Ch|_] when is_integer(Ch) -> Replacement;
+ _ when is_atom(Replacement) -> atom_to_list(Replacement);
+ _ -> false
+ end,
+ if ReplStr == false ->
+ replace_names_in_node1(NodeStr,Ds);
+ true ->
+ case re:replace(NodeStr,atom_to_list(Name),
+ ReplStr,[{return,list}]) of
+ NodeStr -> % no match, proceed
+ replace_names_in_node1(NodeStr,Ds);
+ NodeStr1 ->
+ replace_names_in_node1(NodeStr1,Defs)
+ end
+ end;
+replace_names_in_node1(NodeStr,[]) ->
+ NodeStr.
+
%% global terms that will be used for analysing all other terms in the spec
get_global([{merge_tests,Bool} | Ts], Spec) ->
@@ -361,18 +430,13 @@ get_global([{merge_tests,Bool} | Ts], Spec) ->
get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) ->
get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]});
get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) ->
- Node1 = case Node of
- {N,H} when is_atom(N), is_atom(H) ->
- list_to_atom(atom_to_list(N) ++ "@"
- ++ atom_to_list(H));
- _ ->
- Node
- end,
- get_global(Ts,Spec#testspec{nodes=[{Ref,Node1} |
- lists:keydelete(Node1,2,Refs)]});
-
-get_global([_|Ts],Spec) -> get_global(Ts,Spec);
-get_global([],Spec) -> Spec.
+ get_global(Ts,Spec#testspec{nodes=[{Ref,Node} |
+ lists:keydelete(Node,2,Refs)]});
+
+get_global([_|Ts],Spec) ->
+ get_global(Ts,Spec);
+get_global([],Spec=#testspec{nodes=Ns, alias=As}) ->
+ Spec#testspec{nodes=lists:reverse(Ns), alias=lists:reverse(As)}.
get_absfile(Callback,FullName,#testspec{spec_dir=SpecDir}) ->
% we need to temporary switch to new cwd here, because
@@ -394,41 +458,53 @@ get_absfile(Callback,FullName,#testspec{spec_dir=SpecDir}) ->
FullName
end.
-get_absfile(Split=[[Ch|_]|_],Spec) when is_integer(Ch) ->
- get_absfile(filename:join(Split),Spec);
get_absfile(FullName,#testspec{spec_dir=SpecDir}) ->
File = filename:basename(FullName),
Dir = get_absname(filename:dirname(FullName),SpecDir),
filename:join(Dir,File).
-get_absdir(Split=[[Ch|_]|_],Spec) when is_integer(Ch) ->
- get_absdir(filename:join(Split),Spec);
get_absdir(Dir,#testspec{spec_dir=SpecDir}) ->
get_absname(Dir,SpecDir).
-get_absname(TestDir,SpecDir) ->
- AbsName = filename:absname(TestDir,SpecDir),
- TestDirName = filename:basename(AbsName),
- Path = filename:dirname(AbsName),
- TopDir = filename:basename(Path),
- Path1 =
- case TopDir of
- "." ->
- [_|Rev] = lists:reverse(filename:split(Path)),
- filename:join(lists:reverse(Rev));
- ".." ->
- [_,_|Rev] = lists:reverse(filename:split(Path)),
- filename:join(lists:reverse(Rev));
- _ ->
- Path
- end,
- filename:join(Path1,TestDirName).
+get_absname(Dir,SpecDir) ->
+ AbsName = filename:absname(Dir,SpecDir),
+ shorten_path(AbsName,SpecDir).
+
+shorten_path(Path,SpecDir) ->
+ case shorten_split_path(filename:split(Path),[]) of
+ [] ->
+ [Root|_] = filename:split(SpecDir),
+ Root;
+ Short ->
+ filename:join(Short)
+ end.
+
+shorten_split_path([".."|Path],SoFar) ->
+ shorten_split_path(Path,tl(SoFar));
+shorten_split_path(["."|Path],SoFar) ->
+ shorten_split_path(Path,SoFar);
+shorten_split_path([Dir|Path],SoFar) ->
+ shorten_split_path(Path,[Dir|SoFar]);
+shorten_split_path([],SoFar) ->
+ lists:reverse(SoFar).
%% go through all tests and register all nodes found
get_all_nodes([{suites,Nodes,_,_}|Ts],Spec) when is_list(Nodes) ->
get_all_nodes(Ts,save_nodes(Nodes,Spec));
get_all_nodes([{suites,Node,_,_}|Ts],Spec) ->
get_all_nodes(Ts,save_nodes([Node],Spec));
+get_all_nodes([{groups,[Char|_],_,_,_}|Ts],Spec) when is_integer(Char) ->
+ get_all_nodes(Ts,Spec);
+get_all_nodes([{groups,Nodes,_,_,_}|Ts],Spec) when is_list(Nodes) ->
+ get_all_nodes(Ts,save_nodes(Nodes,Spec));
+get_all_nodes([{groups,Nodes,_,_,_,_}|Ts],Spec) when is_list(Nodes) ->
+ get_all_nodes(Ts,save_nodes(Nodes,Spec));
+get_all_nodes([{groups,_,_,_,{cases,_}}|Ts],Spec) ->
+ get_all_nodes(Ts,Spec);
+get_all_nodes([{groups,Node,_,_,_}|Ts],Spec) ->
+ get_all_nodes(Ts,save_nodes([Node],Spec));
+get_all_nodes([{groups,Node,_,_,_,_}|Ts],Spec) ->
+ get_all_nodes(Ts,save_nodes([Node],Spec));
get_all_nodes([{cases,Nodes,_,_,_}|Ts],Spec) when is_list(Nodes) ->
get_all_nodes(Ts,save_nodes(Nodes,Spec));
get_all_nodes([{cases,Node,_,_,_}|Ts],Spec) ->
@@ -437,80 +513,93 @@ get_all_nodes([{skip_suites,Nodes,_,_,_}|Ts],Spec) when is_list(Nodes) ->
get_all_nodes(Ts,save_nodes(Nodes,Spec));
get_all_nodes([{skip_suites,Node,_,_,_}|Ts],Spec) ->
get_all_nodes(Ts,save_nodes([Node],Spec));
+get_all_nodes([{skip_groups,[Char|_],_,_,_,_}|Ts],Spec) when is_integer(Char) ->
+ get_all_nodes(Ts,Spec);
+get_all_nodes([{skip_groups,Nodes,_,_,_,_}|Ts],Spec) when is_list(Nodes) ->
+ get_all_nodes(Ts,save_nodes(Nodes,Spec));
+get_all_nodes([{skip_groups,Node,_,_,_,_}|Ts],Spec) ->
+ get_all_nodes(Ts,save_nodes([Node],Spec));
+get_all_nodes([{skip_groups,Nodes,_,_,_,_,_}|Ts],Spec) when is_list(Nodes) ->
+ get_all_nodes(Ts,save_nodes(Nodes,Spec));
+get_all_nodes([{skip_groups,Node,_,_,_,_,_}|Ts],Spec) ->
+ get_all_nodes(Ts,save_nodes([Node],Spec));
get_all_nodes([{skip_cases,Nodes,_,_,_,_}|Ts],Spec) when is_list(Nodes) ->
get_all_nodes(Ts,save_nodes(Nodes,Spec));
get_all_nodes([{skip_cases,Node,_,_,_,_}|Ts],Spec) ->
get_all_nodes(Ts,save_nodes([Node],Spec));
-get_all_nodes([_|Ts],Spec) ->
+get_all_nodes([_Other|Ts],Spec) ->
get_all_nodes(Ts,Spec);
get_all_nodes([],Spec) ->
Spec.
-filter_init_terms([{init, InitOptions}|Ts], NewTerms, Spec)->
- filter_init_terms([{init, list_nodes(Spec), InitOptions}|Ts],
- NewTerms, Spec);
-filter_init_terms([{init, NodeRef, InitOptions}|Ts],
- NewTerms, Spec) when is_atom(NodeRef)->
- filter_init_terms([{init, [NodeRef], InitOptions}|Ts], NewTerms, Spec);
-filter_init_terms([{init, NodeRefs, InitOption}|Ts],
- NewTerms, Spec) when is_tuple(InitOption) ->
- filter_init_terms([{init, NodeRefs, [InitOption]}|Ts], NewTerms, Spec);
-filter_init_terms([{init, [NodeRef|NodeRefs], InitOptions}|Ts],
- NewTerms, Spec=#testspec{init=InitData})->
+filter_init_terms([{init,InitOptions}|Ts],NewTerms,Spec) ->
+ filter_init_terms([{init,list_nodes(Spec),InitOptions}|Ts],
+ NewTerms,Spec);
+filter_init_terms([{init,all_nodes,InitOptions}|Ts],NewTerms,Spec) ->
+ filter_init_terms([{init,list_nodes(Spec),InitOptions}|Ts],
+ NewTerms,Spec);
+filter_init_terms([{init,NodeRef,InitOptions}|Ts],
+ NewTerms,Spec) when is_atom(NodeRef) ->
+ filter_init_terms([{init,[NodeRef],InitOptions}|Ts],NewTerms,Spec);
+filter_init_terms([{init,NodeRefs,InitOption}|Ts],
+ NewTerms,Spec) when is_tuple(InitOption) ->
+ filter_init_terms([{init,NodeRefs,[InitOption]}|Ts],NewTerms,Spec);
+filter_init_terms([{init,[NodeRef|NodeRefs],InitOptions}|Ts],
+ NewTerms,Spec=#testspec{init=InitData}) ->
NodeStartOptions =
- case lists:keyfind(node_start, 1, InitOptions) of
- {node_start, NSOptions}->
- case lists:keyfind(callback_module, 1, NSOptions) of
- {callback_module, _Callback}->
+ case lists:keyfind(node_start,1,InitOptions) of
+ {node_start,NSOptions}->
+ case lists:keyfind(callback_module,1,NSOptions) of
+ {callback_module,_Callback}->
NSOptions;
false->
- [{callback_module, ct_slave}|NSOptions]
+ [{callback_module,ct_slave}|NSOptions]
end;
false->
[]
end,
- EvalTerms = case lists:keyfind(eval, 1, InitOptions) of
- {eval, MFA} when is_tuple(MFA)->
+ EvalTerms = case lists:keyfind(eval,1,InitOptions) of
+ {eval,MFA} when is_tuple(MFA) ->
[MFA];
- {eval, MFAs} when is_list(MFAs)->
+ {eval,MFAs} when is_list(MFAs) ->
MFAs;
false->
[]
end,
Node = ref2node(NodeRef,Spec#testspec.nodes),
- InitData2 = add_option({node_start, NodeStartOptions}, Node, InitData, true),
- InitData3 = add_option({eval, EvalTerms}, Node, InitData2, false),
- filter_init_terms([{init, NodeRefs, InitOptions}|Ts],
- NewTerms, Spec#testspec{init=InitData3});
-filter_init_terms([{init, [], _}|Ts], NewTerms, Spec)->
- filter_init_terms(Ts, NewTerms, Spec);
-filter_init_terms([Term|Ts], NewTerms, Spec)->
- filter_init_terms(Ts, [Term|NewTerms], Spec);
-filter_init_terms([], NewTerms, Spec)->
- {lists:reverse(NewTerms), Spec}.
-
-add_option({Key, Value}, Node, List, WarnIfExists) when is_list(Value)->
- OldOptions = case lists:keyfind(Node, 1, List) of
- {Node, Options}->
+ InitData2 = add_option({node_start,NodeStartOptions},Node,InitData,true),
+ InitData3 = add_option({eval,EvalTerms},Node,InitData2,false),
+ filter_init_terms([{init,NodeRefs,InitOptions}|Ts],
+ NewTerms,Spec#testspec{init=InitData3});
+filter_init_terms([{init,[],_}|Ts],NewTerms,Spec) ->
+ filter_init_terms(Ts,NewTerms,Spec);
+filter_init_terms([Term|Ts],NewTerms,Spec) ->
+ filter_init_terms(Ts,[Term|NewTerms],Spec);
+filter_init_terms([],NewTerms,Spec) ->
+ {lists:reverse(NewTerms),Spec}.
+
+add_option({Key,Value},Node,List,WarnIfExists) when is_list(Value) ->
+ OldOptions = case lists:keyfind(Node,1,List) of
+ {Node,Options}->
Options;
false->
[]
end,
- NewOption = case lists:keyfind(Key, 1, OldOptions) of
- {Key, OldOption} when WarnIfExists, OldOption/=[]->
+ NewOption = case lists:keyfind(Key,1,OldOptions) of
+ {Key,OldOption} when WarnIfExists,OldOption/=[]->
io:format("There is an option ~w=~w already "
"defined for node ~p, skipping new ~w~n",
- [Key, OldOption, Node, Value]),
+ [Key,OldOption,Node,Value]),
OldOption;
- {Key, OldOption}->
+ {Key,OldOption}->
OldOption ++ Value;
false->
Value
end,
- lists:keystore(Node, 1, List,
- {Node, lists:keystore(Key, 1, OldOptions, {Key, NewOption})});
-add_option({Key, Value}, Node, List, WarnIfExists)->
- add_option({Key, [Value]}, Node, List, WarnIfExists).
+ lists:keystore(Node,1,List,
+ {Node,lists:keystore(Key,1,OldOptions,{Key,NewOption})});
+add_option({Key,Value},Node,List,WarnIfExists) ->
+ add_option({Key,[Value]},Node,List,WarnIfExists).
save_nodes(Nodes,Spec=#testspec{nodes=NodeRefs}) ->
NodeRefs1 =
@@ -553,7 +642,7 @@ add_tests([{suites,Nodes,Dir,Ss}|Ts],Spec) when is_list(Nodes) ->
add_tests([{suites,Node,Dir,Ss}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_suites(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Ss,Tests, Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
@@ -581,14 +670,14 @@ add_tests([{groups,Nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],
add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Suite,Gs,all,Tests,
Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Suite,Gs,TCs,Tests,
Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
@@ -604,7 +693,7 @@ add_tests([{cases,Nodes,Dir,Suite,Cs}|Ts],Spec) when is_list(Nodes) ->
add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Suite,Cs,Tests, Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
@@ -619,7 +708,7 @@ add_tests([{skip_suites,Nodes,Dir,Ss,Cmt}|Ts],Spec) when is_list(Nodes) ->
add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_suites(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Ss,Cmt,Tests,
Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
@@ -645,14 +734,14 @@ add_tests([{skip_groups,Nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],
add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Suite,Gs,all,Cmt,Tests,
Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Suite,Gs,TCs,Cmt,Tests,
Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
@@ -668,7 +757,7 @@ add_tests([{skip_cases,Nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) when is_list(Nodes) ->
add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes),
- ref2dir(Dir,Spec#testspec.alias),
+ ref2dir(Dir,Spec),
Suite,Cs,Cmt,Tests,Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
@@ -694,6 +783,9 @@ add_tests([{noinput,Bool}|Ts],Spec) ->
add_tests(Ts, Spec#testspec{noinput = Bool});
%% --- handled/errors ---
+add_tests([{define,_,_}|Ts],Spec) -> % handled
+ add_tests(Ts,Spec);
+
add_tests([{alias,_,_}|Ts],Spec) -> % handled
add_tests(Ts,Spec);
@@ -712,10 +804,14 @@ add_tests([{merge_tests, _} | Ts], Spec) -> % handled
%% create one test entry per known node and reinsert
add_tests([Term={Tag,all_nodes,Data}|Ts],Spec) ->
- check_term(Term),
- Tests = [{Tag,Node,Data} || Node <- list_nodes(Spec),
- should_be_added(Tag,Node,Data,Spec)],
- add_tests(Tests++Ts,Spec);
+ case check_term(Term) of
+ valid ->
+ Tests = [{Tag,Node,Data} || Node <- list_nodes(Spec),
+ should_be_added(Tag,Node,Data,Spec)],
+ add_tests(Tests++Ts,Spec);
+ invalid -> % ignore term
+ add_tests(Ts,Spec)
+ end;
%% create one test entry per node in Nodes and reinsert
add_tests([{Tag,[],Data}|Ts],Spec) ->
add_tests([{Tag,all_nodes,Data}|Ts],Spec);
@@ -734,19 +830,27 @@ add_tests([{Tag,NodesOrOther,Data}|Ts],Spec) when is_list(NodesOrOther) ->
add_tests([Term={Tag,NodeOrOther,Data}|Ts],Spec) ->
case is_node(NodeOrOther,Spec#testspec.nodes) of
true ->
- check_term(Term),
- Node = ref2node(NodeOrOther,Spec#testspec.nodes),
- NodeIxData =
- handle_data(Tag,Node,Data,Spec) ++
- update_recorded(Tag,Node,Spec),
- add_tests(Ts,mod_field(Spec,Tag,NodeIxData));
+ case check_term(Term) of
+ valid ->
+ Node = ref2node(NodeOrOther,Spec#testspec.nodes),
+ NodeIxData =
+ update_recorded(Tag,Node,Spec) ++
+ handle_data(Tag,Node,Data,Spec),
+ add_tests(Ts,mod_field(Spec,Tag,NodeIxData));
+ invalid -> % ignore term
+ add_tests(Ts,Spec)
+ end;
false ->
add_tests([{Tag,all_nodes,{NodeOrOther,Data}}|Ts],Spec)
end;
%% this test should be added for all known nodes
-add_tests([{Tag,Data}|Ts],Spec) ->
- add_tests([{Tag,all_nodes,Data}|Ts],Spec);
-
+add_tests([Term={Tag,Data}|Ts],Spec) ->
+ case check_term(Term) of
+ valid ->
+ add_tests([{Tag,all_nodes,Data}|Ts],Spec);
+ invalid ->
+ add_tests(Ts,Spec)
+ end;
%% some other data than a tuple
add_tests([Other|Ts],Spec) ->
case get(relaxed) of
@@ -763,23 +867,32 @@ add_tests([],Spec) -> % done
%% have added something of his/her own, which we'll let pass if relaxed
%% mode is enabled.
check_term(Term) ->
+ Size = size(Term),
[Name|_] = tuple_to_list(Term),
- case lists:keymember(Name,1,valid_terms()) of
- true -> % halt
- throw({error,{bad_term_in_spec,Term}});
- false -> % ignore
- case get(relaxed) of
- true ->
- %% warn if name resembles a CT term
- case resembles_ct_term(Name,size(Term)) of
+ Valid = valid_terms(),
+ case lists:member({Name,Size},Valid) of
+ true ->
+ valid;
+ false ->
+ case lists:keymember(Name,1,Valid) of
+ true -> % halt
+ throw({error,{bad_term_in_spec,Term}});
+ false -> % ignore
+ case get(relaxed) of
true ->
- io:format("~nSuspicious term, please check:~n"
- "~p~n", [Term]);
+ %% warn if name resembles a CT term
+ case resembles_ct_term(Name,size(Term)) of
+ true ->
+ io:format("~nSuspicious term, "
+ "please check:~n"
+ "~p~n", [Term]),
+ invalid;
+ false ->
+ invalid
+ end;
false ->
- ok
- end;
- false ->
- throw({error,{undefined_term_in_spec,Term}})
+ throw({error,{undefined_term_in_spec,Term}})
+ end
end
end.
@@ -1082,22 +1195,16 @@ ref2node(all_nodes,_Refs) ->
ref2node(master,_Refs) ->
master;
ref2node(RefOrNode,Refs) ->
- case string:chr(atom_to_list(RefOrNode),$@) of
- 0 -> % a ref
+ case lists:member($@,atom_to_list(RefOrNode)) of
+ false -> % a ref
case lists:keysearch(RefOrNode,1,Refs) of
{value,{RefOrNode,Node}} ->
Node;
false ->
throw({error,{noderef_missing,RefOrNode}})
end;
- _ -> % a node
- case RefOrNode of
- {TheNode,TheHost} ->
- list_to_atom(atom_to_list(TheNode) ++ "@"
- ++ atom_to_list(TheHost));
- _ ->
- RefOrNode
- end
+ true -> % a node
+ RefOrNode
end.
ref2dir(Ref,Spec) ->
@@ -1111,7 +1218,9 @@ ref2dir(Ref,Refs,Spec) when is_atom(Ref) ->
throw({error,{alias_missing,Ref}})
end;
ref2dir(Dir,_,Spec) when is_list(Dir) ->
- get_absdir(Dir,Spec).
+ get_absdir(Dir,Spec);
+ref2dir(What,_,_) ->
+ throw({error,{invalid_directory_name,What}}).
is_node(What,Nodes) when is_atom(What) ->
is_node([What],Nodes);
@@ -1120,15 +1229,8 @@ is_node([master|_],_Nodes) ->
is_node(What={N,H},Nodes) when is_atom(N), is_atom(H) ->
is_node([What],Nodes);
is_node([What|_],Nodes) ->
- Nodename = case What of
- {N,H} when is_atom(N), is_atom(H) ->
- list_to_atom(atom_to_list(N) ++ "@"
- ++ atom_to_list(H));
- _ ->
- What
- end,
- case lists:keymember(Nodename,1,Nodes) or
- lists:keymember(Nodename,2,Nodes) of
+ case lists:keymember(What,1,Nodes) or
+ lists:keymember(What,2,Nodes) of
true ->
true;
false ->
@@ -1139,6 +1241,7 @@ is_node([],_) ->
valid_terms() ->
[
+ {define,3},
{node,3},
{cover,2},
{cover,3},
@@ -1188,7 +1291,8 @@ valid_terms() ->
{skip_groups,7},
{skip_cases,5},
{skip_cases,6},
- {create_priv_dir,2}
+ {create_priv_dir,2},
+ {create_priv_dir,3}
].
%% this function "guesses" if the user has misspelled a term name
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 560a0b0d5a..9c58d5c36a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -39,13 +39,16 @@ MODULES= \
ct_sequence_1_SUITE \
ct_repeat_1_SUITE \
ct_testspec_1_SUITE \
+ ct_testspec_2_SUITE \
ct_skip_SUITE \
ct_error_SUITE \
ct_test_server_if_1_SUITE \
ct_config_SUITE \
ct_master_SUITE \
ct_misc_1_SUITE \
- ct_hooks_SUITE
+ ct_hooks_SUITE \
+ ct_basic_html_SUITE \
+ ct_auto_compile_SUITE
ERL_FILES= $(MODULES:%=%.erl)
diff --git a/lib/common_test/test/ct_auto_compile_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE.erl
new file mode 100644
index 0000000000..cc546ed30d
--- /dev/null
+++ b/lib/common_test/test/ct_auto_compile_SUITE.erl
@@ -0,0 +1,187 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_auto_compile_SUITE
+%%%
+%%% Description:
+%%%
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_auto_compile_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [ac_flag, ac_spec].
+
+groups() ->
+ [].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+ac_flag(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+ file:copy(filename:join(DataDir, "bad_SUITE.erl"),
+ filename:join(PrivDir, "bad_SUITE.erl")),
+ Suite = filename:join(DataDir, "dummy_SUITE"),
+ compile:file(Suite, [{outdir,PrivDir}]),
+ {Opts,ERPid} = setup([{dir,PrivDir},
+ {auto_compile,false},
+ {label,"ac_flag"}],
+ Config),
+
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(ac_flag,
+ reformat(Events, ?eh),
+ PrivDir,
+ Opts),
+
+ TestEvents = events_to_check(ac_flag),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+ac_spec(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+ file:copy(filename:join(DataDir, "bad_SUITE.erl"),
+ filename:join(PrivDir, "bad_SUITE.erl")),
+ TestSpec = [{label,ac_spec},
+ {auto_compile,false},
+ {suites,PrivDir,all}],
+ FileName = filename:join(?config(priv_dir, Config),"ac_spec.spec"),
+ {ok,Dev} = file:open(FileName, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+
+ {Opts,ERPid} = setup([{spec,FileName}], Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(ac_spec,
+ reformat(Events, ?eh),
+ PrivDir,
+ Opts),
+
+ TestEvents = events_to_check(ac_spec),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+ %reformat(Events, _EH) ->
+ % Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(ac_flag) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(ac_spec) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ].
diff --git a/lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl
new file mode 100644
index 0000000000..6ebcb3570e
--- /dev/null
+++ b/lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl
@@ -0,0 +1,23 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+
+-module(bad_SUITE).
+
+-compile(export_all).
+
+bad_bad_suite
diff --git a/lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl
new file mode 100644
index 0000000000..0b1eafc31d
--- /dev/null
+++ b/lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl
@@ -0,0 +1,130 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+
+-module(dummy_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok,fail,skip].
+
+
+ok(_Config) ->
+ ok.
+
+fail(Config) ->
+ tuple_to_list(Config),
+ ok.
+
+skip(_Config) ->
+ {skip,"should be skipped"}.
diff --git a/lib/common_test/test/ct_basic_html_SUITE.erl b/lib/common_test/test/ct_basic_html_SUITE.erl
new file mode 100644
index 0000000000..a5f2e6197e
--- /dev/null
+++ b/lib/common_test/test/ct_basic_html_SUITE.erl
@@ -0,0 +1,180 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_basic_html_SUITE
+%%%
+%%% Description:
+%%%
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_basic_html_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [basic_flag, basic_spec].
+
+groups() ->
+ [].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+basic_flag(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Suites = [filename:join(DataDir, "babbling_SUITE")],
+ {Opts,ERPid} = setup([{suite,Suites},
+ {basic_html,true},
+ {label,"basic_flag"}],
+ Config),
+
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(basic_flag,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(basic_flag),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+basic_spec(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestSpec = [{label,basic_spec},
+ {basic_html,true},
+ {suites,DataDir,babbling_SUITE}],
+ FileName = filename:join(?config(priv_dir, Config),"basic_spec.spec"),
+ {ok,Dev} = file:open(FileName, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+
+ {Opts,ERPid} = setup([{spec,FileName}], Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(basic_spec,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(basic_spec),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+ %reformat(Events, _EH) ->
+ % Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(basic_flag) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(basic_spec) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ].
diff --git a/lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl b/lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl
new file mode 100644
index 0000000000..d67383c606
--- /dev/null
+++ b/lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl
@@ -0,0 +1,130 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+
+-module(babbling_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok,fail,skip].
+
+
+ok(_Config) ->
+ ok.
+
+fail(Config) ->
+ tuple_to_list(Config),
+ ok.
+
+skip(_Config) ->
+ {skip,"should be skipped"}.
diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl
new file mode 100644
index 0000000000..93f3520f95
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_2_SUITE.erl
@@ -0,0 +1,751 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_testspec_2_SUITE
+%%%
+%%% Description:
+%%% Test test specifications
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_testspec_2_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/src/ct_util.hrl").
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%% suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [basic_compatible_no_nodes,
+ basic_compatible_nodes,
+ unknown_terms,
+ no_merging,
+ multiple_specs,
+ misc_config_terms,
+ define_names_1].
+
+
+%%--------------------------------------------------------------------
+%% VALID TEST SPEC TERMS (R15B02):
+%%
+%% {node,3}
+%% {cover,2}
+%% {cover,3}
+%% {config,2}
+%% {config,3}
+%% {config,4}
+%% {userconfig,2}
+%% {userconfig,3}
+%% {alias,3}
+%% {merge_tests,2}
+%% {logdir,2}
+%% {logdir,3}
+%% {logopts,2}
+%% {logopts,3}
+%% {basic_html,2}
+%% {basic_html,3}
+%% {label,2}
+%% {label,3}
+%% {event_handler,2}
+%% {event_handler,3}
+%% {event_handler,4}
+%% {ct_hooks,2}
+%% {ct_hooks,3}
+%% {enable_builtin_hooks,2}
+%% {noinput,2}
+%% {multiply_timetraps,2}
+%% {multiply_timetraps,3}
+%% {scale_timetraps,2}
+%% {scale_timetraps,3}
+%% {include,2}
+%% {include,3}
+%% {auto_compile,2}
+%% {auto_compile,3}
+%% {stylesheet,2}
+%% {stylesheet,3}
+%% {suites,3}
+%% {suites,4}
+%% {groups,4}
+%% {groups,5}
+%% {groups,6}
+%% {cases,4}
+%% {cases,5}
+%% {skip_suites,4}
+%% {skip_suites,5}
+%% {skip_groups,5}
+%% {skip_groups,6}
+%% {skip_groups,7}
+%% {skip_cases,5}
+%% {skip_cases,6}
+%% {create_priv_dir,2}
+%%
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+basic_compatible_no_nodes(_Config) ->
+
+ AliasDir1 = "../tests/to1",
+ AliasDir2 = "../tests/to2",
+ CfgDir1 = "../cfgs/to1/x.cfg",
+ CfgDir2 = ["../cfgs/to2/x.cfg","../cfgs/to2/y.cfg"],
+ LogDir = "../logs",
+ IncludeDir1 = "../../include",
+ IncludeDir2 = ["../tests/to1/include","../tests/to2/include"],
+
+ Spec =
+ [
+ {label,"basic_compatible_no_nodes"},
+ {alias,to1,AliasDir1},
+ {alias,to2,AliasDir2},
+ {config,CfgDir1},
+ {config,CfgDir2},
+ {userconfig,{?MODULE,"cfg_str1"}},
+ {userconfig,{?MODULE,"cfg_str2"}},
+ {logdir,LogDir},
+ {logopts,[no_nl]},
+ {event_handler,evh1,[1]},
+ {event_handler,[evh2,evh3],[[2,3]]},
+ {ct_hooks,[{cth_mod1,[]}]},
+ {ct_hooks,[{cth_mod2,[]}]},
+ {multiply_timetraps,2},
+ {include,IncludeDir1},
+ {include,IncludeDir2},
+ {suites,to1,[x_SUITE]},
+ {groups,to1,y_SUITE,[g1,g2]},
+ {cases,to1,y_SUITE,[tc1,tc2]},
+ {skip_suites,to1,z_SUITE,"skipped"},
+ {suites,to2,[x_SUITE,y_SUITE]},
+ {skip_groups,to2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,to2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "basic_compatible_no_nodes.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
+ Alias1V = get_absdir(filename:join(SpecDir,AliasDir1)),
+ Alias2V = get_absdir(filename:join(SpecDir,AliasDir2)),
+ CFGs = [{Node,get_absdir(filename:join(SpecDir,CfgDir))} ||
+ CfgDir <- [CfgDir1 | CfgDir2]],
+ Incls = [{Node,get_absdir(filename:join(SpecDir,IncludeDir))} ||
+ IncludeDir <- [IncludeDir1 | IncludeDir2]],
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,Node}],
+ init = [],
+ label = [{Node,"basic_compatible_no_nodes"}],
+ logdir = [{Node,LogDirV},"."],
+ logopts = [{Node,[no_nl]}],
+ basic_html = [],
+ cover = [],
+ config = CFGs,
+ userconfig = [{Node,{?MODULE,"cfg_str1"}},
+ {Node,{?MODULE,"cfg_str2"}}],
+ event_handler = [{Node,evh1,[1]},
+ {Node,evh2,[[2,3]]},
+ {Node,evh3,[[2,3]]}],
+ ct_hooks = [{Node,{cth_mod1,[]}},
+ {Node,{cth_mod2,[]}}],
+ enable_builtin_hooks = true,
+ noinput = false,
+ include = Incls,
+ auto_compile = [],
+ stylesheet = [],
+ multiply_timetraps = [{Node,2}],
+ scale_timetraps = [],
+ create_priv_dir = [],
+ alias = [{to1,Alias1V},{to2,Alias2V}],
+ tests = [{{Node,Alias1V},
+ [{x_SUITE,[all]},
+ {y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+ {z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node,Alias2V},
+ [{x_SUITE,[all,
+ {{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]},
+ {y_SUITE,[all,
+ {tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}],
+ merge_tests = true},
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+basic_compatible_nodes(_Config) ->
+
+ Node1 = node1@host1,
+ Node2 = node2@host2,
+ TODir1 = "../tests/to1",
+ TODir2 = "../tests/to2",
+ CfgDir1 = "../cfgs/to1/x.cfg",
+ CfgDir2 = ["../cfgs/to2/x.cfg","../cfgs/to2/y.cfg"],
+ LogDir = "../logs",
+ MasterLogDir = "../master_logs",
+ IncludeDir1 = "../../include",
+ IncludeDir2 = ["../tests/to1/include","../tests/to2/include"],
+
+ Spec =
+ [
+ {node,n1,Node1},
+ {node,n2,Node2},
+ {init,[n1],[{node_start,[{callback_module,cbm}]}]},
+ {init,n2,[{node_start,[]}]},
+ {init,all_nodes,{eval,{mod,func,[]}}},
+ {label,"basic_compatible_nodes"},
+ {label,n1,basic_compatible_nodes_1},
+ {config,n1,CfgDir1},
+ {config,n2,CfgDir2},
+ {userconfig,{?MODULE,"cfg_str1"}},
+ {userconfig,{?MODULE,"cfg_str2"}},
+ {logdir,all_nodes,LogDir},
+ {logdir,master,MasterLogDir},
+ {logopts,node2@host2,[no_nl]},
+ {event_handler,master,evh1,[1]},
+ {event_handler,[n1,n2],[evh2,evh3],[[2,3]]},
+ {ct_hooks,all_nodes,[{cth_mod1,[]}]},
+ {ct_hooks,[{cth_mod2,[]}]},
+ {multiply_timetraps,node1@host1,2},
+ {include,n1,IncludeDir1},
+ {include,[n1,n2],IncludeDir2},
+ {suites,n1,TODir1,[x_SUITE]},
+ {groups,n1,TODir1,y_SUITE,[g1,g2]},
+ {cases,n1,TODir1,y_SUITE,[tc1,tc2]},
+ {skip_suites,n1,TODir1,z_SUITE,"skipped"},
+ {suites,n2,TODir2,[x_SUITE,y_SUITE]},
+ {skip_groups,n2,TODir2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,n2,TODir2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "basic_compatible_nodes.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
+ MasterLogDirV = get_absdir(filename:join(SpecDir,"../master_logs")),
+ TO1V = get_absdir(filename:join(SpecDir,TODir1)),
+ TO2V = get_absdir(filename:join(SpecDir,TODir2)),
+ CFGs = [{Node1,get_absdir(filename:join(SpecDir,CfgDir1))} |
+ [{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]],
+ Incls = [{Node1,get_absdir(filename:join(SpecDir,IncludeDir1))} |
+ [{Node1,get_absdir(filename:join(SpecDir,IncludeDir))} ||
+ IncludeDir <- IncludeDir2] ++
+ [{Node2,get_absdir(filename:join(SpecDir,IncludeDir))} ||
+ IncludeDir <- IncludeDir2]],
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
+ init = [{Node1,[{node_start,[{callback_module,cbm}]},
+ {eval,[{mod,func,[]}]}]},
+ {Node2,[{node_start,[{callback_module,ct_slave}]},
+ {eval,[{mod,func,[]}]}]},
+ {Node,[{node_start,[]},
+ {eval,[{mod,func,[]}]}]}],
+ label = [{Node,"basic_compatible_nodes"},
+ {Node2,"basic_compatible_nodes"},
+ {Node1,basic_compatible_nodes_1}],
+ logdir = [{Node,LogDirV},{Node1,LogDirV},{Node2,LogDirV},
+ {master,MasterLogDirV},"."],
+ logopts = [{Node2,[no_nl]}],
+ basic_html = [],
+ cover = [],
+ config = CFGs,
+ userconfig = [{Node,{?MODULE,"cfg_str1"}},
+ {Node1,{?MODULE,"cfg_str1"}},
+ {Node2,{?MODULE,"cfg_str1"}},
+ {Node,{?MODULE,"cfg_str2"}},
+ {Node1,{?MODULE,"cfg_str2"}},
+ {Node2,{?MODULE,"cfg_str2"}}],
+ event_handler = [{master,evh1,[1]},
+ {Node1,evh2,[[2,3]]},
+ {Node1,evh3,[[2,3]]},
+ {Node2,evh2,[[2,3]]},
+ {Node2,evh3,[[2,3]]}],
+ ct_hooks = [{Node,{cth_mod1,[]}},
+ {Node1,{cth_mod1,[]}},
+ {Node2,{cth_mod1,[]}},
+ {Node,{cth_mod2,[]}},
+ {Node1,{cth_mod2,[]}},
+ {Node2,{cth_mod2,[]}}],
+ enable_builtin_hooks = true,
+ noinput = false,
+ include = Incls,
+ auto_compile = [],
+ stylesheet = [],
+ multiply_timetraps = [{Node1,2}],
+ scale_timetraps = [],
+ create_priv_dir = [],
+ tests = [{{Node1,TO1V},
+ [{x_SUITE,[all]},
+ {y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+ {z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[all,
+ {{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]},
+ {y_SUITE,[all,
+ {tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}],
+ merge_tests = true},
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+unknown_terms(Config) ->
+ PrivDir = ?config(priv_dir, Config),
+
+ Spec1 = [{suites,PrivDir,all},
+ {userdata,"I've got news for you"}],
+ {error,{undefined_term_in_spec,{userdata,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec1, false)),
+ true = is_record(ct_testspec:collect_tests_from_list(Spec1, true),
+ testspec),
+
+ Spec2 = [{logdir,{logdir,PrivDir}}],
+ {error,{invalid_directory_name,_}} =
+ (catch ct_testspec:collect_tests_from_list(Spec2, false)),
+
+ Spec3 = [{suite,PrivDir,all}],
+ {error,{undefined_term_in_spec,{suite,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec3, false)),
+ true = is_record(ct_testspec:collect_tests_from_list(Spec3, true), testspec),
+
+ Spec4 = [{suites,PrivDir,all},
+ {skip_suites,PrivDir,x_SUITE}],
+ {error,{bad_term_in_spec,{skip_suites,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec4, false)),
+ {error,{bad_term_in_spec,{skip_suites,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec4, true)),
+
+ Spec5 = [{configs,all_nodes,PrivDir}],
+ {error,{undefined_term_in_spec,{configs,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec5, false)),
+ true = is_record(ct_testspec:collect_tests_from_list(Spec5, true), testspec),
+
+ ok.
+
+%%%-----------------------------------------------------------------
+%%%
+no_merging(_Config) ->
+ Node1 = node1@host1,
+ Node2 = node2@host2,
+ TODir1 = "../tests/to1",
+ TODir2 = "../tests/to2",
+ Spec =
+ [
+ {merge_tests,false},
+ {node,n1,Node1},
+ {node,n2,Node2},
+ {suites,n1,TODir1,[x_SUITE]},
+ {groups,n1,TODir1,y_SUITE,[g1,g2]},
+ {cases,n1,TODir1,y_SUITE,[tc1,tc2]},
+ {skip_suites,n1,TODir1,z_SUITE,"skipped"},
+ {suites,n2,TODir2,[x_SUITE,y_SUITE]},
+ {skip_groups,n2,TODir2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,n2,TODir2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "no_merging.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ TO1V = get_absdir(filename:join(SpecDir,TODir1)),
+ TO2V = get_absdir(filename:join(SpecDir,TODir2)),
+
+ Verify = #testspec{merge_tests = false,
+ spec_dir = SpecDir,
+ nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
+ tests = [{{Node1,TO1V},
+ [{x_SUITE,[all]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[{g1,all},{g2,all}]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[tc1,tc2]}]},
+ {{Node1,TO1V},
+ [{z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[{{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[{tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}]},
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+multiple_specs(_Config) ->
+ Node1 = node1@host1,
+ Node2 = node2@host2,
+ TODir1 = "../tests/to1",
+ TODir2 = "../tests/to2",
+ CfgDir1 = "../cfgs/to1/x.cfg",
+ CfgDir2 = ["../cfgs/to2/x.cfg","../cfgs/to2/y.cfg"],
+ LogDir = "../logs",
+ Spec1 =
+ [
+ {node,n1,Node1},
+ {node,n2,Node2},
+ {alias,to1,TODir1},
+ {alias,to2,TODir2},
+ {label,"multiple_specs1"},
+ {config,n1,CfgDir1},
+ {config,n2,CfgDir2},
+ {logdir,all_nodes,LogDir}
+ ],
+ Spec2 =
+ [
+ {merge_tests,false},
+ {label,"multiple_specs2"},
+ {suites,n1,TODir1,[x_SUITE]},
+ {groups,n1,TODir1,y_SUITE,[g1,g2]},
+ {cases,n1,TODir1,y_SUITE,[tc1,tc2]},
+ {skip_suites,n1,TODir1,z_SUITE,"skipped"},
+ {suites,n2,TODir2,[x_SUITE,y_SUITE]},
+ {skip_groups,n2,TODir2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,n2,TODir2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+ SpecFile1 = ct_test_support:write_testspec(Spec1,SpecDir,
+ "multiple_specs.1.spec"),
+ SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir,
+ "multiple_specs.2.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2],
+ false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ TO1V = get_absdir(filename:join(SpecDir,TODir1)),
+ TO2V = get_absdir(filename:join(SpecDir,TODir2)),
+ CFGs = [{Node1,get_absdir(filename:join(SpecDir,CfgDir1))} |
+ [{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]],
+ LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
+
+ Verify = #testspec{merge_tests = false,
+ spec_dir = SpecDir,
+ nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
+ alias = [{to1,TO1V},{to2,TO2V}],
+ label = [{Node,"multiple_specs1"},
+ {Node1,"multiple_specs1"},
+ {Node2,"multiple_specs1"}],
+ logdir = [{Node,LogDirV},{Node1,LogDirV},{Node2,LogDirV},"."],
+ config = CFGs,
+ tests = [{{Node1,TO1V},
+ [{x_SUITE,[all]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[{g1,all},{g2,all}]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[tc1,tc2]}]},
+ {{Node1,TO1V},
+ [{z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[{{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[{tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}]},
+
+ verify_result(Verify,FileResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+misc_config_terms(_Config) ->
+ CfgDir = "../cfgs/to1",
+
+ Spec =
+ [{node,x,n1@h1},{node,y,n2@h2},
+
+ {config,CfgDir,"a.cfg"},
+ {config,n1@h1,CfgDir,"b.cfg"},
+ {config,all_nodes,CfgDir,"c.cfg"},
+ {config,all_nodes,filename:join(CfgDir,"d.cfg")},
+
+ {basic_html,true},
+ {basic_html,n1@h1,false},
+ {basic_html,n2@h2,true},
+
+ {enable_builtin_hooks,false},
+
+ {noinput,true},
+
+ {auto_compile,false},
+ {auto_compile,n1@h1,true},
+ {auto_compile,n2@h2,false},
+
+ {stylesheet,"../css"},
+ {stylesheet,n1@h1,"./n1/css"},
+ {stylesheet,n2@h2,"./n2/css"},
+
+ {create_priv_dir,[auto_per_tc]},
+ {create_priv_dir,n1@h1,[manual_per_tc]},
+ {create_priv_dir,n2@h2,[auto_per_run]}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "misc_config_terms.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ CfgA = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "a.cfg")),
+ CfgB = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "b.cfg")),
+ CfgC = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "c.cfg")),
+ CfgD = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "d.cfg")),
+ CSS = get_absdir(filename:join(SpecDir,"../css")),
+ CSS1 = get_absdir(filename:join(SpecDir,"./n1/css")),
+ CSS2 = get_absdir(filename:join(SpecDir,"./n2/css")),
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,Node},{x,n1@h1},{y,n2@h2}],
+ basic_html = [{Node,true},{n1@h1,false},{n2@h2,true}],
+ config = [{Node,CfgA},
+ {n1@h1,CfgA},
+ {n2@h2,CfgA},
+ {n1@h1,CfgB},
+ {Node,CfgC},
+ {n1@h1,CfgC},
+ {n2@h2,CfgC},
+ {Node,CfgD},
+ {n1@h1,CfgD},
+ {n2@h2,CfgD}],
+ enable_builtin_hooks = false,
+ noinput = true,
+ auto_compile = [{Node,false},
+ {n1@h1,true},
+ {n2@h2,false}],
+ stylesheet = [{Node,CSS},
+ {n1@h1,CSS1},
+ {n2@h2,CSS2}],
+ create_priv_dir = [{Node,[auto_per_tc]},
+ {n1@h1,[manual_per_tc]},
+ {n2@h2,[auto_per_run]}]
+ },
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+define_names_1(_Config) ->
+ Spec =
+ [
+ {define,'HOST','eniac'},
+ {define,'NODE1',testnode1},
+ {define,'NODE2',testnode2},
+ {define,'NODES',['NODE1@HOST',
+ 'NODE2@HOST']},
+ {define,'TOPDIR',".."},
+ {define,'TO1',"to1"},
+ {define,'TO2',"to2"},
+ {define,'LOGDIR',"'TOPDIR'/logdir"},
+ {define,'LOGDIR1',"'TOPDIR'/logdir1"},
+ {define,'LOGDIR2',"'TOPDIR'/logdir2"},
+ {define,'CFGDIR',"'TOPDIR'/cfgs"},
+ {define,'CFGFILES',["cfgX","cfgY"]},
+ {define,'TESTDIR',"'TOPDIR'/test"},
+ {define,'TO1DIR',"'TESTDIR'/'TO1'"},
+ {define,'TO2DIR',"'TESTDIR'/'TO2'"},
+ {define,'EXSUITE',ex_SUITE},
+ {define,'EXGRS',[g1,g2]},
+
+ {logdir,'LOGDIR'},
+ {logdir,'NODE1@HOST','LOGDIR1'},
+ {logdir,'NODE2@HOST','LOGDIR2'},
+
+ {config,["a.cfg","b.cfg"]},
+ {config,'NODES',"./'CFGDIR'/c.cfg"},
+ {config,'CFGDIR',["d.cfg","e.cfg"]},
+ {config,'NODE2@HOST','CFGDIR','CFGFILES'},
+
+ {suites,'NODE1@HOST','TO1DIR',all},
+ {suites,'NODES','TO2DIR',all},
+
+ {groups,'TO1DIR','EXSUITE','EXGRS'}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "define_names_1.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ N = node(),
+ N1 = testnode1@eniac,
+ N2 = testnode2@eniac,
+ Join = fun(Dir) -> shorten_path(filename:join(SpecDir,Dir),SpecDir) end,
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,N2},
+ {undefined,N1},
+ {undefined,N}],
+ config = [{N2,Join("a.cfg")},{N2,Join("b.cfg")},
+ {N1,Join("a.cfg")},{N1,Join("b.cfg")},
+ {N,Join("a.cfg")},{N,Join("b.cfg")},
+ {N1,Join("../cfgs/c.cfg")},{N2,Join("../cfgs/c.cfg")},
+ {N2,Join("../cfgs/d.cfg")},{N2,Join("../cfgs/e.cfg")},
+ {N1,Join("../cfgs/d.cfg")},{N1,Join("../cfgs/e.cfg")},
+ {N,Join("../cfgs/d.cfg")},{N,Join("../cfgs/e.cfg")},
+ {N2,Join("../cfgs/cfgX")},{N2,Join("../cfgs/cfgY")}],
+ logdir = [{N,Join("../logdir")},
+ {N1,Join("../logdir1")},
+ {N2,Join("../logdir2")},
+ "."],
+ tests = [{{N1,Join("../test/to1")},[{all,[all]}]},
+ {{N1,Join("../test/to2")},[{all,[all]}]},
+ {{N2,Join("../test/to2")},[{all,[all]}]},
+ {{N2,Join("../test/to1")},
+ [{ex_SUITE,[{g1,all},{g2,all}]}]},
+ {{N,Join("../test/to1")},
+ [{ex_SUITE,[{g1,all},{g2,all}]}]}]
+ },
+ verify_result(Verify,ListResult,FileResult).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+verify_result(Verify,ListResult,FileResult) ->
+ {_,TSLTuples} = rec2proplist(ListResult),
+ {_,TSFTuples} = rec2proplist(FileResult),
+ {_,VTuples} = rec2proplist(Verify),
+ VResult =
+ (catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) ->
+ {TSL,TSF};
+ ({Tag,Val},{[{_Tag,TSLVal}|_TSL],[{Tag,Val}|_TSF]}) ->
+ exit({ts_list_mismatch,Tag,Val,TSLVal});
+ ({Tag,Val},{[{Tag,Val}|_TSL],[{_Tag,TSFVal}|_TSF]}) ->
+ exit({ts_file_mismatch,Tag,Val,TSFVal});
+ ({Tag,Val},{_,[{_Tag,TSFVal}|_TSF]}) ->
+ exit({ts_mismatch,Tag,Val,TSFVal})
+ end, {TSLTuples,TSFTuples}, VTuples)),
+ case VResult of
+ {'EXIT',Reason} ->
+ ct:fail(Reason);
+ _ ->
+ ok
+ end,
+ ok.
+
+
+check_parameter(S="cfg_str1") ->
+ {ok,{config,S}};
+check_parameter(S="cfg_str2") ->
+ {ok,{config,S}}.
+read_config(S) ->
+ {ok,[{cfg,S}]}.
+
+rec2proplist(E={error,_What}) ->
+ exit({invalid_testspec_record,E});
+rec2proplist(Rec) ->
+ [RecName|RecList] = tuple_to_list(Rec),
+ FieldNames =
+ if RecName == testspec ->
+ record_info(fields, testspec);
+ true ->
+ undefined
+ end,
+ {RecName,combine_names_and_vals(FieldNames,RecList)}.
+
+combine_names_and_vals([FN|FNs], [V|Vs]) ->
+ [{FN,V} | combine_names_and_vals(FNs, Vs)];
+combine_names_and_vals([], []) ->
+ [];
+combine_names_and_vals(_, _) ->
+ [].
+
+get_absdir(Dir) ->
+ shorten_path(filename:absname(Dir),Dir).
+
+shorten_path(Path,SpecDir) ->
+ case shorten_split_path(filename:split(Path),[]) of
+ [] ->
+ [Root|_] = filename:split(SpecDir),
+ Root;
+ Short ->
+ filename:join(Short)
+ end.
+
+shorten_split_path([".."|Path],SoFar) ->
+ shorten_split_path(Path,tl(SoFar));
+shorten_split_path(["."|Path],SoFar) ->
+ shorten_split_path(Path,SoFar);
+shorten_split_path([Dir|Path],SoFar) ->
+ shorten_split_path(Path,[Dir|SoFar]);
+shorten_split_path([],SoFar) ->
+ lists:reverse(SoFar).