diff options
Diffstat (limited to 'lib/test_server/src')
26 files changed, 1294 insertions, 868 deletions
diff --git a/lib/test_server/src/Makefile b/lib/test_server/src/Makefile index 35bbad3c22..6a26ee2933 100644 --- a/lib/test_server/src/Makefile +++ b/lib/test_server/src/Makefile @@ -3,16 +3,17 @@ # # Copyright Ericsson AB 1996-2013. All Rights Reserved. # -# The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved online at http://www.erlang.org/. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # %CopyrightEnd% # diff --git a/lib/test_server/src/configure.in b/lib/test_server/src/configure.in index 8398825d95..001de72a1e 100644 --- a/lib/test_server/src/configure.in +++ b/lib/test_server/src/configure.in @@ -4,16 +4,17 @@ dnl %CopyrightBegin% dnl dnl Copyright Ericsson AB 1997-2014. All Rights Reserved. dnl -dnl The contents of this file are subject to the Erlang Public License, -dnl Version 1.1, (the "License"); you may not use this file except in -dnl compliance with the License. You should have received a copy of the -dnl Erlang Public License along with this software. If not, it can be -dnl retrieved online at http://www.erlang.org/. -dnl -dnl Software distributed under the License is distributed on an "AS IS" -dnl basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -dnl the License for the specific language governing rights and limitations -dnl under the License. +dnl Licensed under the Apache License, Version 2.0 (the "License"); +dnl you may not use this file except in compliance with the License. +dnl You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. dnl dnl %CopyrightEnd% dnl diff --git a/lib/test_server/src/erl2html2.erl b/lib/test_server/src/erl2html2.erl index 50dbbb82ee..e69383acea 100644 --- a/lib/test_server/src/erl2html2.erl +++ b/lib/test_server/src/erl2html2.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2015. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -43,7 +44,7 @@ convert(File, Dest, InclPath) -> "<html>\n" "<head>\n" "<meta http-equiv=\"Content-Type\" content=\"text/html;" - "charset=",html_encoding(Encoding),"\"/>\n" + "charset=",html_encoding(Encoding),"\"/></meta>\n" "<title>", to_raw_list(File,Encoding), "</title>\n" "</head>\n\n" "<body bgcolor=\"white\" text=\"black\"" @@ -109,26 +110,26 @@ parse_file(File, InclPath) -> Error end. -parse_preprocessed_file(Epp,File,InCorrectFile) -> +parse_preprocessed_file(Epp, File, InCorrectFile) -> case epp:parse_erl_form(Epp) of {ok,Form} -> case Form of {attribute,_,file,{File,_}} -> - parse_preprocessed_file(Epp,File,true); + parse_preprocessed_file(Epp, File, true); {attribute,_,file,{_OtherFile,_}} -> - parse_preprocessed_file(Epp,File,false); - {function,L,F,A,Cs} when InCorrectFile -> - {CLs,LastCL} = find_clause_lines(Cs, []), + parse_preprocessed_file(Epp, File, false); + {function,L,F,A,Cs} when InCorrectFile -> + {CLs,LastCL} = find_clause_lines(Cs, []), %% tl(CLs) cause we know the start line already - [{atom_to_list(F),A,L,LastCL} | tl(CLs)] ++ - parse_preprocessed_file(Epp,File,true); + [{atom_to_list(F),A,get_line(L),LastCL} | tl(CLs)] ++ + parse_preprocessed_file(Epp, File, true); _ -> - parse_preprocessed_file(Epp,File,InCorrectFile) + parse_preprocessed_file(Epp, File, InCorrectFile) end; {error,Reason={_L,epp,{undefined,_Macro,none}}} -> throw({error,Reason,InCorrectFile}); {error,_Reason} -> - parse_preprocessed_file(Epp,File,InCorrectFile); + parse_preprocessed_file(Epp, File, InCorrectFile); {eof,_Location} -> [] end. @@ -147,10 +148,10 @@ parse_non_preprocessed_file(Epp, File, Location) -> case epp_dodger:parse_form(Epp, Location) of {ok,Tree,Location1} -> try erl_syntax:revert(Tree) of - {function,L,F,A,Cs} -> - {CLs,LastCL} = find_clause_lines(Cs, []), + {function,L,F,A,Cs} -> + {CLs,LastCL} = find_clause_lines(Cs, []), %% tl(CLs) cause we know the start line already - [{atom_to_list(F),A,L,LastCL} | tl(CLs)] ++ + [{atom_to_list(F),A,get_line(L),LastCL} | tl(CLs)] ++ parse_non_preprocessed_file(Epp, File, Location1); _ -> parse_non_preprocessed_file(Epp, File, Location1) @@ -163,21 +164,28 @@ parse_non_preprocessed_file(Epp, File, Location) -> [] end. +get_line(Anno) -> + erl_anno:line(Anno). + %%%----------------------------------------------------------------- %%% Find the line number of the last expression in the function find_clause_lines([{clause,CL,_Params,_Op,Exprs}], CLs) -> % last clause try tuple_to_list(lists:last(Exprs)) of - [_Type,ExprLine | _] -> - {lists:reverse([{clause,CL}|CLs]), ExprLine}; + [_Type,ExprLine | _] when is_integer(ExprLine) -> + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(ExprLine)}; + [tree,_ | Exprs1] -> + find_clause_lines([{clause,CL,undefined,undefined,Exprs1}], CLs); + [macro,{_var,ExprLine,_MACRO} | _] when is_integer(ExprLine) -> + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(ExprLine)}; _ -> - {lists:reverse([{clause,CL}|CLs]), CL} + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(CL)} catch _:_ -> - {lists:reverse([{clause,CL}|CLs]), CL} + {lists:reverse([{clause,get_line(CL)}|CLs]), get_line(CL)} end; find_clause_lines([{clause,CL,_Params,_Op,_Exprs} | Cs], CLs) -> - find_clause_lines(Cs, [{clause,CL}|CLs]). + find_clause_lines(Cs, [{clause,get_line(CL)}|CLs]). %%%----------------------------------------------------------------- %%% Add a link target for each line and one for each function definition. @@ -185,18 +193,18 @@ build_html(SFd,DFd,Encoding,FuncsAndCs) -> build_html(SFd,DFd,Encoding,file:read_line(SFd),1,FuncsAndCs, false,undefined). -%% function start line found -build_html(SFd,DFd,Enc,{ok,Str},L0,[{F,A,L0,LastL}|FuncsAndCs], - _IsFuncDef,_FAndLastL) -> - FALink = test_server_ctrl:uri_encode(F++"-"++integer_to_list(A),utf8), - file:write(DFd,["<a name=\"",to_raw_list(FALink,Enc),"\"/>"]), - build_html(SFd,DFd,Enc,{ok,Str},L0,FuncsAndCs,true,{F,LastL}); %% line of last expression in function found build_html(SFd,DFd,Enc,{ok,Str},LastL,FuncsAndCs,_IsFuncDef,{F,LastL}) -> LastLineLink = test_server_ctrl:uri_encode(F++"-last_expr",utf8), file:write(DFd,["<a name=\"", to_raw_list(LastLineLink,Enc),"\"/>"]), build_html(SFd,DFd,Enc,{ok,Str},LastL,FuncsAndCs,true,undefined); +%% function start line found +build_html(SFd,DFd,Enc,{ok,Str},L0,[{F,A,L0,LastL}|FuncsAndCs], + _IsFuncDef,_FAndLastL) -> + FALink = test_server_ctrl:uri_encode(F++"-"++integer_to_list(A),utf8), + file:write(DFd,["<a name=\"",to_raw_list(FALink,Enc),"\"/>"]), + build_html(SFd,DFd,Enc,{ok,Str},L0,FuncsAndCs,true,{F,LastL}); build_html(SFd,DFd,Enc,{ok,Str},L,[{clause,L}|FuncsAndCs], _IsFuncDef,FAndLastL) -> build_html(SFd,DFd,Enc,{ok,Str},L,FuncsAndCs,true,FAndLastL); diff --git a/lib/test_server/src/test_server.app.src b/lib/test_server/src/test_server.app.src index 173f7075db..334be8109d 100644 --- a/lib/test_server/src/test_server.app.src +++ b/lib/test_server/src/test_server.app.src @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2009-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% @@ -32,7 +33,7 @@ test_server_break_process]}, {applications, [kernel,stdlib]}, {env, []}, - {runtime_dependencies, ["tools-2.6.14","stdlib-2.0","runtime_tools-1.8.14", - "observer-2.0","kernel-3.0","inets-5.10", - "syntax_tools-1.6.16","erts-6.0"]}]}. + {runtime_dependencies, ["tools-2.8","stdlib-2.5","runtime_tools-1.8.16", + "observer-2.1","kernel-4.0","inets-6.0", + "syntax_tools-1.7","erts-7.0"]}]}. diff --git a/lib/test_server/src/test_server.appup.src b/lib/test_server/src/test_server.appup.src index 42c6fe2e46..7c4aa630ae 100644 --- a/lib/test_server/src/test_server.appup.src +++ b/lib/test_server/src/test_server.appup.src @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2014. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% {"%VSN%", diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 1c3352550b..919526c5d7 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -1,21 +1,21 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2014. All Rights Reserved. +%% Copyright Ericsson AB 1996-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 %% -%% %CopyrightEnd% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% +%% %CopyrightEnd% -module(test_server). -define(DEFAULT_TIMETRAP_SECS, 60). @@ -63,13 +63,11 @@ init_target_info() -> [$.|Emu] = code:objfile_extension(), {_, OTPRel} = init:script_id(), - TestServerDir = filename:absname(filename:dirname(code:which(?MODULE))), #target_info{os_family=test_server_sup:get_os_family(), os_type=os:type(), version=erlang:system_info(version), system_version=erlang:system_info(system_version), root_dir=code:root_dir(), - test_server_dir=TestServerDir, emulator=Emu, otp_release=OTPRel, username=test_server_sup:get_username(), @@ -178,68 +176,35 @@ module_names(Beams) -> do_cover_compile(Modules) -> cover:start(), - pmap1(fun(M) -> do_cover_compile1(M) end,lists:usort(Modules)), + Sticky = prepare_cover_compile(Modules,[]), + R = cover:compile_beam(Modules), + [warn_compile(Error) || Error <- R,element(1,Error)=/=ok], + [code:stick_mod(M) || M <- Sticky], ok. -do_cover_compile1(M) -> +warn_compile({error,{Reason,Module}}) -> + io:fwrite("\nWARNING: Could not cover compile ~ts: ~p\n", + [Module,{error,Reason}]). + +%% Make sure all modules are loaded and unstick if sticky +prepare_cover_compile([M|Ms],Sticky) -> case {code:is_sticky(M),code:is_loaded(M)} of {true,_} -> code:unstick_mod(M), - case cover:compile_beam(M) of - {ok,_} -> - ok; - Error -> - io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n", - [M,Error]) - end, - code:stick_mod(M); + prepare_cover_compile(Ms,[M|Sticky]); {false,false} -> case code:load_file(M) of {module,_} -> - do_cover_compile1(M); + prepare_cover_compile([M|Ms],Sticky); Error -> - io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]) + io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]), + prepare_cover_compile(Ms,Sticky) end; {false,_} -> - case cover:compile_beam(M) of - {ok,_} -> - ok; - Error -> - io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n", - [M,Error]) - end - end. - -pmap1(Fun,List) -> - NTot = length(List), - NProcs = erlang:system_info(schedulers) * 2, - NPerProc = (NTot div NProcs) + 1, - - {[],Pids} = - lists:foldr( - fun(_,{L,Ps}) -> - {L1,L2} = if length(L)>=NPerProc -> lists:split(NPerProc,L); - true -> {L,[]} % last chunk - end, - {P,_Ref} = - spawn_monitor(fun() -> - exit(lists:map(Fun,L1)) - end), - {L2,[P|Ps]} - end, - {List,[]}, - lists:seq(1,NProcs)), - collect(Pids,[]). - -collect([],Acc) -> - lists:append(Acc); -collect([Pid|Pids],Acc) -> - receive - {'DOWN', _Ref, process, Pid, Result} -> - %% collect(lists:delete(Pid,Pids),[Result|Acc]) - collect(Pids,[Result|Acc]) - end. - + prepare_cover_compile(Ms,Sticky) + end; +prepare_cover_compile([],Sticky) -> + Sticky. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop) -> @@ -269,45 +234,40 @@ collect([Pid|Pids],Acc) -> %% after the test is completed. cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop}) -> io:fwrite(user, "Cover analysing... ", []), - DetailsFun = + {ATFOk,ATFFail} = case Analyse of details -> case cover:export(filename:join(Dir,"all.coverdata")) of ok -> - fun(M) -> - OutFile = filename:join(Dir, - atom_to_list(M) ++ - ".COVER.html"), - case cover:analyse_to_file(M,OutFile,[html]) of - {ok,_} -> - {file,OutFile}; - Error -> - Error - end - end; + {result,Ok1,Fail1} = + cover:analyse_to_file(Modules,[{outdir,Dir},html]), + {lists:map(fun(OutFile) -> + M = list_to_atom( + filename:basename( + filename:rootname(OutFile, + ".COVER.html") + ) + ), + {M,{file,OutFile}} + end, Ok1), + lists:map(fun({Reason,M}) -> + {M,{error,Reason}} + end, Fail1)}; Error -> - fun(_) -> Error end + {[],lists:map(fun(M) -> {M,Error} end, Modules)} end; overview -> case cover:export(filename:join(Dir,"all.coverdata")) of ok -> - fun(_) -> undefined end; + {[],lists:map(fun(M) -> {M,undefined} end, Modules)}; Error -> - fun(_) -> Error end + {[],lists:map(fun(M) -> {M,Error} end, Modules)} end end, - R = pmap2( - fun(M) -> - case cover:analyse(M,module) of - {ok,{M,{Cov,NotCov}}} -> - {M,{Cov,NotCov,DetailsFun(M)}}; - Err -> - io:fwrite(user, - "\nWARNING: Analysis failed for ~w. Reason: ~p\n", - [M,Err]), - {M,Err} - end - end, Modules), + {result,AOk,AFail} = cover:analyse(Modules,module), + R0 = merge_analysis_results(AOk,ATFOk++ATFFail,[]) ++ + [{M,{error,Reason}} || {Reason,M} <- AFail], + R = lists:sort(R0), io:fwrite(user, "done\n\n", []), case Stop of @@ -320,19 +280,15 @@ cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop}) -> end, R. -pmap2(Fun,List) -> - Collector = self(), - Pids = lists:map(fun(E) -> - spawn(fun() -> - Collector ! {res,self(),Fun(E)} - end) - end, List), - lists:map(fun(Pid) -> - receive - {res,Pid,Res} -> - Res - end - end, Pids). +merge_analysis_results([{M,{Cov,NotCov}}|T],ATF,Acc) -> + case lists:keytake(M,1,ATF) of + {value,{_,R},ATF1} -> + merge_analysis_results(T,ATF1,[{M,{Cov,NotCov,R}}|Acc]); + false -> + merge_analysis_results(T,ATF,Acc) + end; +merge_analysis_results([],_,Acc) -> + Acc. do_cover_for_node(Node,CoverFunc) -> do_cover_for_node(Node,CoverFunc,true). @@ -453,11 +409,9 @@ run_test_case_apply(Mod, Func, Args, Name, RunInit, TimetrapData) -> Ref = make_ref(), Pid = spawn_link( - fun() -> - run_test_case_eval(Mod, Func, Args, Name, Ref, - RunInit, TimetrapData, - LogOpts, TCCallback) - end), + run_test_case_eval_fun(Mod, Func, Args, Name, Ref, + RunInit, TimetrapData, + LogOpts, TCCallback)), put(test_server_detected_fail, []), St = #st{ref=Ref,pid=Pid,mf={Mod,Func},last_known_loc=unknown, status=starting,ret_val=[],comment="",timeout=infinity, @@ -760,10 +714,10 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) -> Starter ! {self(),{call_end_conf,Data,ok}} end); true -> - do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) + do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal) end. -do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> +do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal) -> EndConfProc = fun() -> process_flag(trap_exit,true), % to catch timetraps @@ -771,16 +725,25 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> EndConfApply = fun() -> timetrap(TVal), - try apply(Mod,end_per_testcase,[Func,Conf]) of + %% We can't handle fails or skips here + %% (neither input nor output). The error can + %% be read from Conf though (tc_status). + EndConf = + case do_init_tc_call(Mod,{end_per_testcase,Func}, + [Conf], + {TCExitReason,[Conf]}) of + {_,[EPTCInit]} when is_list(EPTCInit) -> + EPTCInit; + _ -> + Conf + end, + try apply(Mod,end_per_testcase,[Func,EndConf]) of _ -> ok catch - _:Why -> + _:Error -> timer:sleep(1), - group_leader() ! {printout,12, - "WARNING! " - "~w:end_per_testcase(~w, ~p)" - " crashed!\n\tReason: ~p\n", - [Mod,Func,Conf,Why]} + print_end_conf_result(Mod,Func,Conf, + "crashed",Error) end, Supervisor ! {self(),end_conf} end, @@ -789,10 +752,7 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> {Pid,end_conf} -> Starter ! {self(),{call_end_conf,Data,ok}}; {'EXIT',Pid,Reason} -> - group_leader() ! {printout,12, - "WARNING! ~w:end_per_testcase(~w, ~p)" - " failed!\n\tReason: ~p\n", - [Mod,Func,Conf,Reason]}, + print_end_conf_result(Mod,Func,Conf,"failed",Reason), Starter ! {self(),{call_end_conf,Data,{error,Reason}}}; {'EXIT',_OtherPid,Reason} -> %% Probably the parent - not much to do about that @@ -801,28 +761,55 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) -> end, spawn_link(EndConfProc). -spawn_fw_call(Mod,{init_per_testcase,Func},CurrConf,Pid, - {timetrap_timeout,TVal}=Why, - Loc,SendTo) -> +print_end_conf_result(Mod,Func,Conf,Cause,Error) -> + Str2Print = + fun(NoHTML) when NoHTML == stdout; NoHTML == major -> + io_lib:format("WARNING! " + "~w:end_per_testcase(~w, ~tp)" + " ~s!\n\tReason: ~tp\n", + [Mod,Func,Conf,Cause,Error]); + (minor) -> + ErrorStr = test_server_ctrl:escape_chars(Error), + io_lib:format("WARNING! " + "~w:end_per_testcase(~w, ~tp)" + " ~s!\n\tReason: ~ts\n", + [Mod,Func,Conf,Cause,ErrorStr]) + end, + group_leader() ! {printout,12,Str2Print}. + + +spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid, + Why,Loc,SendTo) -> FwCall = fun() -> Skip = {skip,{failed,{Mod,init_per_testcase,Why}}}, %% if init_per_testcase fails, the test case %% should be skipped - try do_end_tc_call(Mod,Func, {Pid,Skip,[CurrConf]}, Why) of + try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why), + do_init_tc_call(Mod,{end_per_testcase,Func}, + [CurrConf],{ok,[CurrConf]}), + do_end_tc_call(Mod,{end_per_testcase,Func}, + {Pid,Skip,[CurrConf]}, Why) end of _ -> ok catch _:FwEndTCErr -> exit({fw_notify_done,end_tc,FwEndTCErr}) end, + Time = case Why of + {timetrap_timeout,TVal} -> TVal/1000; + _ -> died + end, + group_leader() ! {printout,12, + "ERROR! ~w:init_per_testcase(~w, ~p)" + " failed!\n\tReason: ~tp\n", + [Mod,Func,CurrConf,Why]}, %% finished, report back - SendTo ! {self(),fw_notify_done, - {TVal/1000,Skip,Loc,[],undefined}} + SendTo ! {self(),fw_notify_done,{Time,Skip,Loc,[],undefined}} end, spawn_link(FwCall); -spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid, - {timetrap_timeout,TVal}=Why,_Loc,SendTo) -> +spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid, + Why,_Loc,SendTo) -> FwCall = fun() -> {RetVal,Report} = @@ -836,24 +823,38 @@ spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid, E = {failed,{Mod,end_per_testcase,Why}}, {Result,E} end, - group_leader() ! {printout,12, - "WARNING! ~w:end_per_testcase(~w, ~p)" - " failed!\n\tReason: timetrap timeout" - " after ~w ms!\n", [Mod,Func,EndConf,TVal]}, - FailLoc = proplists:get_value(tc_fail_loc, EndConf), - try do_end_tc_call(Mod,Func, - {Pid,Report,[EndConf]}, Why) of + {Time,Warn} = + case Why of + {timetrap_timeout,TVal} -> + group_leader() ! + {printout,12, + "WARNING! ~w:end_per_testcase(~w, ~p)" + " failed!\n\tReason: timetrap timeout" + " after ~w ms!\n", [Mod,Func,EndConf,TVal]}, + W = "<font color=\"red\">" + "WARNING: end_per_testcase timed out!</font>", + {TVal/1000,W}; + _ -> + group_leader() ! + {printout,12, + "WARNING! ~w:end_per_testcase(~w, ~p)" + " failed!\n\tReason: ~tp\n", + [Mod,Func,EndConf,Why]}, + W = "<font color=\"red\">" + "WARNING: end_per_testcase failed!</font>", + {died,W} + end, + try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of _ -> ok catch _:FwEndTCErr -> exit({fw_notify_done,end_tc,FwEndTCErr}) end, - Warn = "<font color=\"red\">" - "WARNING: end_per_testcase timed out!</font>", + FailLoc = proplists:get_value(tc_fail_loc, EndConf), %% finished, report back (if end_per_testcase fails, a warning %% should be printed as part of the comment) SendTo ! {self(),fw_notify_done, - {TVal/1000,RetVal,FailLoc,[],Warn}} + {Time,RetVal,FailLoc,[],Warn}} end, spawn_link(FwCall); @@ -876,10 +877,12 @@ spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) -> spawn_link(FwCall); spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> - Func1 = case Func of - {_InitOrEndPerTC,F} -> F; - F -> F - end, + {Func1,EndTCFunc} = case Func of + CF when CF == init_per_suite; CF == end_per_suite; + CF == init_per_group; CF == end_per_group -> + {CF,CF}; + TC -> {TC,{end_per_testcase,TC}} + end, FwCall = fun() -> try fw_error_notify(Mod,Func1,[], @@ -891,8 +894,7 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) -> FwErrorNotifyErr}) end, Conf = [{tc_status,{failed,Error}}|CurrConf], - try do_end_tc_call(Mod,Func1, - {Pid,Error,[Conf]},Error) of + try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of _ -> ok catch _:FwEndTCErr -> @@ -949,6 +951,16 @@ job_proxy_msgloop() -> end, job_proxy_msgloop(). +-spec run_test_case_eval_fun(_, _, _, _, _, _, _, _, _) -> + fun(() -> no_return()). +run_test_case_eval_fun(Mod, Func, Args, Name, Ref, RunInit, + TimetrapData, LogOpts, TCCallback) -> + fun () -> + run_test_case_eval(Mod, Func, Args, Name, Ref, + RunInit, TimetrapData, + LogOpts, TCCallback) + end. + %% A test case is known to have failed if it returns {'EXIT', _} tuple, %% or sends a message {failed, File, Line} to it's group_leader @@ -959,32 +971,38 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit, Where = [{Mod,Func}], put(test_server_loc, Where), - FWInitResult = test_server_sup:framework_call(init_tc,[Mod,Func,Args0], - {ok,Args0}), + FWInitFunc = case RunInit of + run_init -> {init_per_testcase,Func}; + _ -> Func + end, + + FWInitResult0 = do_init_tc_call(Mod,FWInitFunc,Args0,{ok,Args0}), + set_tc_state(running), {{Time,Value},Loc,Opts} = - case FWInitResult of + case FWInitResult0 of {ok,Args} -> run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback); Error = {error,_Reason} -> - NewResult = do_end_tc_call(Mod,Func, {Error,Args0}, + NewResult = do_end_tc_call(Mod,FWInitFunc, {Error,Args0}, {auto_skip,{failed,Error}}), {{0,NewResult},Where,[]}; {fail,Reason} -> - Conf = [{tc_status,{failed,Reason}} | hd(Args0)], + Conf = [{tc_status,{failed,Reason}} | hd(Args0)], fw_error_notify(Mod, Func, Conf, Reason), - NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf]}, + NewResult = do_end_tc_call(Mod,FWInitFunc, + {{error,Reason},[Conf]}, {fail,Reason}), {{0,NewResult},Where,[]}; Skip = {SkipType,_Reason} when SkipType == skip; SkipType == skipped -> - NewResult = do_end_tc_call(Mod,Func, + NewResult = do_end_tc_call(Mod,FWInitFunc, {Skip,Args0}, Skip), {{0,NewResult},Where,[]}; AutoSkip = {auto_skip,_Reason} -> %% special case where a conf case "pretends" to be skipped NewResult = - do_end_tc_call(Mod,Func, {AutoSkip,Args0}, AutoSkip), + do_end_tc_call(Mod,FWInitFunc, {AutoSkip,Args0}, AutoSkip), {{0,NewResult},Where,[]} end, exit({Ref,Time,Value,Loc,Opts}). @@ -999,31 +1017,41 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> SkipType == skipped -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}}|hd(Args)], - NewRes = do_end_tc_call(Mod,Func, + NewRes = do_end_tc_call(Mod,{init_per_testcase,Func}, {Skip,[Conf]}, Skip), {{0,NewRes},Line,[]}; {skip_and_save,Reason,SaveCfg} -> Line = get_loc(), Conf = [{tc_status,{skipped,Reason}}, {save_config,SaveCfg}|hd(Args)], - NewRes = do_end_tc_call(Mod,Func, {{skip,Reason},[Conf]}, + NewRes = do_end_tc_call(Mod,{init_per_testcase,Func}, + {{skip,Reason},[Conf]}, {skip,Reason}), {{0,NewRes},Line,[]}; FailTC = {fail,Reason} -> % user fails the testcase EndConf = [{tc_status,{failed,Reason}} | hd(Args)], fw_error_notify(Mod, Func, EndConf, Reason), - NewRes = do_end_tc_call(Mod,Func, + NewRes = do_end_tc_call(Mod,{init_per_testcase,Func}, {{error,Reason},[EndConf]}, FailTC), {{0,NewRes},[{Mod,Func}],[]}; {ok,NewConf} -> - %% call user callback function if defined - NewConf1 = - user_callback(TCCallback, Mod, Func, init, NewConf), - %% save current state in controller loop - set_tc_state(tc, NewConf1), - %% execute the test case - {{T,Return},Loc} = {ts_tc(Mod,Func,[NewConf1]), get_loc()}, + IPTCEndRes = do_end_tc_call(Mod,{init_per_testcase,Func}, + {ok,[NewConf]}, NewConf), + {{T,Return},Loc,NewConf1} = + if not is_list(IPTCEndRes) -> + %% received skip or fail, not config + {{0,IPTCEndRes},undefined,NewConf}; + true -> + %% call user callback function if defined + NewConfUC = + user_callback(TCCallback, Mod, Func, + init, IPTCEndRes), + %% save current state in controller loop + set_tc_state(tc, NewConfUC), + %% execute the test case + {ts_tc(Mod,Func,[NewConfUC]),get_loc(),NewConfUC} + end, {EndConf,TSReturn,FWReturn} = case Return of {E,TCError} when E=='EXIT' ; E==failed -> @@ -1049,36 +1077,47 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> %% call user callback function if defined EndConf1 = user_callback(TCCallback, Mod, Func, 'end', EndConf), + + %% We can't handle fails or skips here + EndConf2 = + case do_init_tc_call(Mod,{end_per_testcase,Func}, + [EndConf1],{ok,[EndConf1]}) of + {ok,[EPTCInitRes]} when is_list(EPTCInitRes) -> + EPTCInitRes; + _ -> + EndConf1 + end, + %% update current state in controller loop - {FWReturn1,TSReturn1,EndConf2} = - case end_per_testcase(Mod, Func, EndConf1) of + {FWReturn1,TSReturn1,EndConf3} = + case end_per_testcase(Mod, Func, EndConf2) of SaveCfg1={save_config,_} -> {FWReturn,TSReturn, [SaveCfg1|lists:keydelete(save_config,1, - EndConf1)]}; + EndConf2)]}; {fail,ReasonToFail} -> %% user has failed the testcase - fw_error_notify(Mod, Func, EndConf1, + fw_error_notify(Mod, Func, EndConf2, ReasonToFail), {{error,ReasonToFail}, {failed,ReasonToFail}, - EndConf1}; + EndConf2}; {failed,{_,end_per_testcase,_}} = Failure when FWReturn == ok -> %% unexpected termination in end_per_testcase %% report this as the result to the framework - {Failure,TSReturn,EndConf1}; + {Failure,TSReturn,EndConf2}; _ -> %% test case result should be reported to %% framework no matter the status of %% end_per_testcase - {FWReturn,TSReturn,EndConf1} + {FWReturn,TSReturn,EndConf2} end, %% clear current state in controller loop - case do_end_tc_call(Mod,Func, - {FWReturn1,[EndConf2]}, TSReturn1) of + case do_end_tc_call(Mod,{end_per_testcase,Func}, + {FWReturn1,[EndConf3]}, TSReturn1) of {failed,Reason} = NewReturn -> - fw_error_notify(Mod,Func,EndConf2, Reason), + fw_error_notify(Mod,Func,EndConf3, Reason), {{T,NewReturn},[{Mod,Func}],[]}; NewReturn -> {{T,NewReturn},Loc,[]} @@ -1104,7 +1143,38 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) -> {{T,Return2},Loc,Opts} end. +do_init_tc_call(Mod, Func, Res, Return) -> + test_server_sup:framework_call(init_tc,[Mod,Func,Res],Return). + +do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) -> + case Return of + {NOk,_} when NOk == auto_skip; NOk == fail; + NOk == skip ; NOk == skipped -> + {_,Args} = Res, + IPTCEndRes = + case do_end_tc_call1(Mod, IPTC, Res, Return) of + IPTCEndConfig when is_list(IPTCEndConfig) -> + IPTCEndConfig; + _ -> + Args + end, + EPTCInitRes = + case do_init_tc_call(Mod,{end_per_testcase,Func}, + IPTCEndRes,Return) of + {ok,EPTCInitConfig} when is_list(EPTCInitConfig) -> + {Return,EPTCInitConfig}; + _ -> + Return + end, + do_end_tc_call1(Mod, {end_per_testcase,Func}, + EPTCInitRes, Return); + _Ok -> + do_end_tc_call1(Mod, IPTC, Res, Return) + end; do_end_tc_call(Mod, Func, Res, Return) -> + do_end_tc_call1(Mod, Func, Res, Return). + +do_end_tc_call1(Mod, Func, Res, Return) -> FwMod = os:getenv("TEST_SERVER_FRAMEWORK"), Ref = make_ref(), if FwMod == "ct_framework" ; FwMod == "undefined"; FwMod == false -> @@ -1243,7 +1313,7 @@ do_init_per_testcase(Mod, Args) -> Bad -> group_leader() ! {printout,12, "ERROR! init_per_testcase has returned " - "bad elements in Config: ~p\n",[Bad]}, + "bad elements in Config: ~tp\n",[Bad]}, {skip,{failed,{Mod,init_per_testcase,bad_return}}} end; {fail,_Reason}=Res -> @@ -1261,25 +1331,33 @@ do_init_per_testcase(Mod, Args) -> throw:Other -> set_loc(erlang:get_stacktrace()), Line = get_loc(), - FormattedLoc = test_server_sup:format_loc(Line), - group_leader() ! {printout,12, - "ERROR! init_per_testcase thrown!\n" - "\tLocation: ~ts\n\tReason: ~p\n", - [FormattedLoc, Other]}, + print_init_conf_result(Line,"thrown",Other), {skip,{failed,{Mod,init_per_testcase,Other}}}; _:Reason0 -> Stk = erlang:get_stacktrace(), Reason = {Reason0,Stk}, set_loc(Stk), Line = get_loc(), - FormattedLoc = test_server_sup:format_loc(Line), - group_leader() ! {printout,12, - "ERROR! init_per_testcase crashed!\n" - "\tLocation: ~ts\n\tReason: ~p\n", - [FormattedLoc,Reason]}, + print_init_conf_result(Line,"crashed",Reason), {skip,{failed,{Mod,init_per_testcase,Reason}}} end. +print_init_conf_result(Line,Cause,Reason) -> + FormattedLoc = test_server_sup:format_loc(Line), + Str2Print = + fun(NoHTML) when NoHTML == stdout; NoHTML == major -> + io_lib:format("ERROR! init_per_testcase ~s!\n" + "\tLocation: ~p\n\tReason: ~tp\n", + [Cause,Line,Reason]); + (minor) -> + ReasonStr = test_server_ctrl:escape_chars(Reason), + io_lib:format("ERROR! init_per_testcase ~s!\n" + "\tLocation: ~ts\n\tReason: ~ts\n", + [Cause,FormattedLoc,ReasonStr]) + end, + group_leader() ! {printout,12,Str2Print}. + + end_per_testcase(Mod, Func, Conf) -> case erlang:function_exported(Mod,end_per_testcase,2) of true -> @@ -1314,12 +1392,7 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> comment(io_lib:format("~ts<font color=\"red\">" "WARNING: ~w thrown!" "</font>\n",[Comment0,EndFunc])), - group_leader() ! {printout,12, - "WARNING: ~w thrown!\n" - "Reason: ~p\n" - "Line: ~ts\n", - [EndFunc, Other, - test_server_sup:format_loc(get_loc())]}, + print_end_tc_warning(EndFunc,Other,"thrown",get_loc()), {failed,{Mod,end_per_testcase,Other}}; Class:Reason -> Stk = erlang:get_stacktrace(), @@ -1336,15 +1409,25 @@ do_end_per_testcase(Mod,EndFunc,Func,Conf) -> comment(io_lib:format("~ts<font color=\"red\">" "WARNING: ~w crashed!" "</font>\n",[Comment0,EndFunc])), - group_leader() ! {printout,12, - "WARNING: ~w crashed!\n" - "Reason: ~p\n" - "Line: ~ts\n", - [EndFunc, Reason, - test_server_sup:format_loc(get_loc())]}, + print_end_tc_warning(EndFunc,Reason,"crashed",get_loc()), {failed,{Mod,end_per_testcase,Why}} end. +print_end_tc_warning(EndFunc,Reason,Cause,Loc) -> + FormattedLoc = test_server_sup:format_loc(Loc), + Str2Print = + fun(NoHTML) when NoHTML == stdout; NoHTML == major -> + io_lib:format("WARNING: ~w ~s!\n" + "Reason: ~tp\nLine: ~p\n", + [EndFunc,Cause,Reason,Loc]); + (minor) -> + ReasonStr = test_server_ctrl:escape_chars(Reason), + io_lib:format("WARNING: ~w ~s!\n" + "Reason: ~ts\nLine: ~ts\n", + [EndFunc,Cause,ReasonStr,FormattedLoc]) + end, + group_leader() ! {printout,12,Str2Print}. + get_loc() -> get(test_server_loc). @@ -1445,7 +1528,7 @@ lookup_config(Key,Config) -> %% stack traces. If the name changes, get_loc/1 must be updated! %% ts_tc(M, F, A) -> - Before = erlang:now(), + Before = erlang:monotonic_time(), Result = try apply(M, F, A) catch @@ -1465,12 +1548,8 @@ ts_tc(M, F, A) -> {'EXIT',Reason} end end, - After = erlang:now(), - Elapsed = - (element(1,After)*1000000000000 - +element(2,After)*1000000+element(3,After)) - - (element(1,Before)*1000000000000 - +element(2,Before)*1000000+element(3,Before)), + After = erlang:monotonic_time(), + Elapsed = erlang:convert_time_unit(After-Before, native, micro_seconds), {Elapsed, Result}. set_loc(Stk) -> @@ -1889,7 +1968,7 @@ time_ms_check(Other) -> time_ms_apply(Func, TCPid, MultAndScale) -> {_,GL} = process_info(TCPid, group_leader), WhoAmI = self(), % either TC or IO server - T0 = now(), + T0 = erlang:monotonic_time(), UserTTSup = spawn(fun() -> user_timetrap_supervisor(Func, WhoAmI, TCPid, @@ -1922,7 +2001,8 @@ user_timetrap_supervisor(Func, Spawner, TCPid, GL, T0, MultAndScale) -> receive {UserTT,Result} -> demonitor(MonRef, [flush]), - Elapsed = trunc(timer:now_diff(now(), T0) / 1000), + T1 = erlang:monotonic_time(), + Elapsed = erlang:convert_time_unit(T1-T0, native, milli_seconds), try time_ms_check(Result) of TimeVal -> %% this is the new timetrap value to set (return value @@ -1990,7 +2070,7 @@ update_user_timetraps(TCPid, StartTime) -> proplists:delete(TCPid, UserTTs)), proceed; {OtherUserTTSup,OtherStartTime} -> - case timer:now_diff(OtherStartTime, StartTime) of + case OtherStartTime - StartTime of Diff when Diff >= 0 -> ignore; _ -> @@ -2085,10 +2165,19 @@ get_timetrap_info(TCPid, SendToServer) -> Timers -> case [Info || {Handle,Pid,Info} <- Timers, Pid == TCPid, Handle /= infinity] of - [I|_] -> - I; + [{TVal,true}|_] -> + {TVal,{true,test_server:timetrap_scale_factor()}}; + [{TVal,false}|_] -> + {TVal,{false,1}}; [] when SendToServer == true -> - tc_supervisor_req({get_timetrap_info,TCPid}); + case tc_supervisor_req({get_timetrap_info,TCPid}) of + {TVal,true} -> + {TVal,{true,test_server:timetrap_scale_factor()}}; + {TVal,false} -> + {TVal,{false,1}}; + Error -> + Error + end; [] -> undefined end @@ -2445,24 +2534,15 @@ is_release_available(Release) -> %% run_on_shielded_node(Fun, CArgs) when is_function(Fun), is_list(CArgs) -> - {A,B,C} = now(), - Name = "shielded_node-" ++ integer_to_list(A) ++ "-" ++ integer_to_list(B) - ++ "-" ++ integer_to_list(C), + Nr = erlang:unique_integer([positive]), + Name = "shielded_node-" ++ integer_to_list(Nr), Node = case start_node(Name, slave, [{args, "-hidden " ++ CArgs}]) of {ok, N} -> N; Err -> fail({failed_to_start_shielded_node, Err}) end, Master = self(), Ref = make_ref(), - Slave = spawn(Node, - fun () -> - start_job_proxy(), - receive - Ref -> - Master ! {Ref, Fun()} - end, - receive after infinity -> infinity end - end), + Slave = spawn(Node, start_job_proxy_fun(Master, Fun)), MRef = erlang:monitor(process, Slave), Slave ! Ref, receive @@ -2477,6 +2557,17 @@ run_on_shielded_node(Fun, CArgs) when is_function(Fun), is_list(CArgs) -> end end. +-spec start_job_proxy_fun(_, _) -> fun(() -> no_return()). +start_job_proxy_fun(Master, Fun) -> + fun () -> + start_job_proxy(), + receive + Ref -> + Master ! {Ref, Fun()} + end, + receive after infinity -> infinity end + end. + %% Return true if Name or node() is a shielded node is_shielded(Name) -> case {cast_to_list(Name),atom_to_list(node())} of @@ -2506,9 +2597,8 @@ is_cover(Name) -> %% A filename of the form <Stem><Number> is generated, and the %% function checks that that file doesn't already exist. temp_name(Stem) -> - {A,B,C} = erlang:now(), - RandomNum = A bxor B bxor C, - RandomName = Stem ++ integer_to_list(RandomNum), + Num = erlang:unique_integer([positive]), + RandomName = Stem ++ integer_to_list(Num), {ok,Files} = file:list_dir(filename:dirname(Stem)), case lists:member(RandomName,Files) of true -> @@ -2538,11 +2628,7 @@ appup_test(App) -> %% Checks wether the module is natively compiled or not. is_native(Mod) -> - case catch Mod:module_info(native_addresses) of - [_|_] -> true; - _Other -> false - end. - + (catch Mod:module_info(native)) =:= true. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% comment(String) -> ok diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 68b03a5987..e0975ab744 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2002-2014. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -76,7 +77,7 @@ -export([handle_call/3, handle_cast/2, handle_info/2]). -export([do_test_cases/4]). -export([do_spec/2, do_spec_list/2]). --export([xhtml/2]). +-export([xhtml/2, escape_chars/1]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -99,7 +100,7 @@ -define(last_link, "last_link"). -define(last_test, "last_test"). -define(html_ext, ".html"). --define(now, erlang:now()). +-define(now, os:timestamp()). -define(void_fun, fun() -> ok end). -define(mod_result(X), if X == skip -> skipped; @@ -1204,19 +1205,14 @@ init_tester(Mod, Func, Args, Dir, Name, {_,_,MinLev}=Levels, report_severe_error(Reason) -> test_server_sup:framework_call(report, [severe_error,Reason]). -%% timer:tc/3 -ts_tc(M, F, A) -> - Before = ?now, - Val = (catch apply(M, F, A)), - After = ?now, - Elapsed = elapsed_time(Before, After), - {Elapsed,Val}. - -elapsed_time(Before, After) -> - (element(1,After)*1000000000000 + - element(2,After)*1000000 + element(3,After)) - - (element(1,Before)*1000000000000 + - element(2,Before)*1000000 + element(3,Before)). +ts_tc(M,F,A) -> + Before = erlang:monotonic_time(), + Result = (catch apply(M, F, A)), + After = erlang:monotonic_time(), + Elapsed = erlang:convert_time_unit(After-Before, + native, + micro_seconds), + {Elapsed, Result}. start_extra_tools(ExtraTools) -> start_extra_tools(ExtraTools, []). @@ -1812,26 +1808,32 @@ start_minor_log_file1(Mod, Func, LogDir, AbsName, MFA) -> io:put_chars(Fd, "<pre>\n"), SrcListing = downcase(atom_to_list(Mod)) ++ ?src_listing_ext, - - {Info,Arity} = - if Func == init_per_suite; Func == end_per_suite -> - {"Config function: ", 1}; - Func == init_per_group; Func == end_per_group -> - {"Config function: ", 2}; - true -> - {"Test case: ", 1} - end, - case {filelib:is_file(filename:join(LogDir, SrcListing)), - lists:member(no_src, get(test_server_logopts))} of - {true,false} -> - print(Lev, Info ++ "<a href=\"~ts#~ts\">~w:~w/~w</a> " - "(click for source code)\n", - [uri_encode(SrcListing), - uri_encode(atom_to_list(Func)++"-1",utf8), - Mod,Func,Arity]); + case get_fw_mod(?MODULE) of + Mod when Func == error_in_suite -> + ok; _ -> - print(Lev, Info ++ "~w:~w/~w\n", [Mod,Func,Arity]) + {Info,Arity} = + if Func == init_per_suite; Func == end_per_suite -> + {"Config function: ", 1}; + Func == init_per_group; Func == end_per_group -> + {"Config function: ", 2}; + true -> + {"Test case: ", 1} + end, + + case {filelib:is_file(filename:join(LogDir, SrcListing)), + lists:member(no_src, get(test_server_logopts))} of + {true,false} -> + print(Lev, ["$tc_html", + Info ++ "<a href=\"~ts#~ts\">~w:~w/~w</a> " + "(click for source code)\n"], + [uri_encode(SrcListing), + uri_encode(atom_to_list(Func)++"-1",utf8), + Mod,Func,Arity]); + _ -> + print(Lev, ["$tc_html",Info ++ "~w:~w/~w\n"], [Mod,Func,Arity]) + end end, AbsName. @@ -2025,7 +2027,7 @@ add_init_and_end_per_suite([{conf,_,_,{Mod,_}}=Case|Cases], LastMod, PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod, NextRef, FwMod)]; add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod) - when element(1,SkipCase) == skip_case -> + when element(1,SkipCase) == skip_case; element(1,SkipCase) == auto_skip_case-> [SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) -> [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)]; @@ -2490,7 +2492,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, file:set_cwd(filename:dirname(get(test_server_dir))), After = ?now, Before = get(test_server_parallel_start_time), - Elapsed = elapsed_time(Before, After)/1000000, + Elapsed = timer:now_diff(After, Before)/1000000, put(test_server_total_time, Elapsed), {false,tl(Mode0),undefined,Elapsed, update_status(Ref, OkSkipFail, Status)}; @@ -2499,7 +2501,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, %% parallel group (io buffering is active) OkSkipFail = wait_for_cases(Ref), queue_test_case_io(Ref, self(), 0, Mod, Func), - Elapsed = elapsed_time(conf_start(Ref, Mode0),?now)/1000000, + Elapsed = timer:now_diff(?now, conf_start(Ref, Mode0))/1000000, case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2516,12 +2518,12 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, %% this is an end conf for a non-parallel group that's not %% nested under a parallel group, so no need to buffer io {false,tl(Mode0),undefined, - elapsed_time(conf_start(Ref, Mode0),?now)/1000000, Status}; + timer:now_diff(?now, conf_start(Ref, Mode0))/1000000, Status}; {Ref,_} -> %% this is an end conf for a non-parallel group nested under %% a parallel group (io buffering is active) queue_test_case_io(Ref, self(), 0, Mod, Func), - Elapsed = elapsed_time(conf_start(Ref, Mode0),?now)/1000000, + Elapsed = timer:now_diff(?now, conf_start(Ref, Mode0))/1000000, case CurrIOHandler of {Ref,_} -> %% current_io_handler was set by start conf of this @@ -2576,7 +2578,7 @@ run_test_cases_loop([{conf,Ref,Props,{Mod,Func}}|_Cases]=Cs0, %% 1. check the TS_RANDOM_SEED env variable %% 2. check random_seed in process state %% 3. use value provided with shuffle option - %% 4. use now() values for seed + %% 4. use timestamp() values for seed case os:getenv("TS_RANDOM_SEED") of Undef when Undef == false ; Undef == "undefined" -> case get(test_server_random_seed) of @@ -3710,8 +3712,8 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, RunDir = filename:dirname(MinorName), Ext = if Num == 0 -> - {_,S,Us} = now(), - lists:flatten(io_lib:format(".~w.~w", [S,Us])); + Int = erlang:unique_integer([positive,monotonic]), + lists:flatten(io_lib:format(".cfg.~w", [Int])); true -> lists:flatten(io_lib:format(".~w", [Num])) end, @@ -3740,7 +3742,10 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, true -> ok end, - print(minor, "Config value:\n\n ~tp\n", [Args2Print]), + + print(minor, + escape_chars(io_lib:format("Config value:\n\n ~tp\n", [Args2Print])), + []), print(minor, "Current directory is ~tp\n", [Cwd]), GrNameStr = case GrName of @@ -3755,7 +3760,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, "<td>" ++ Col0 ++ "~w" ++ Col1 ++ "</td>" "<td>" ++ Col0 ++ "~ts" ++ Col1 ++ "</td>" "<td><a href=\"~ts\">~w</a></td>" - "<td><a href=\"~ts#top\"><</a> <a href=\"~ts#end\">></a></td>", + "<td><a href=\"~ts#top\"><</a> <a href=\"~ts#end\">></a></td>", [num2str(Num),fw_name(Mod),GrNameStr,EncMinorBase,Func, EncMinorBase,EncMinorBase]), @@ -3932,7 +3937,7 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, {ReportTag,Reason1}}]), - ReasonStr = reason_to_string(Reason1), + ReasonStr = escape_chars(reason_to_string(Reason1)), ReasonStr1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ReasonStr,[$\n])]), ReasonStr2 = @@ -3951,8 +3956,8 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time, "<td>~ts~ts</td></tr>\n", [Time,Color,ReasonStr2,Comment1]), FormatLoc = test_server_sup:format_loc(Loc), - print(minor, "=== location ~ts", [FormatLoc]), - print(minor, "=== reason = ~ts", [ReasonStr1]), + print(minor, "=== Location: ~ts", [FormatLoc]), + print(minor, "=== Reason: ~ts", [ReasonStr1]), Ret; progress(failed, CaseNum, Mod, Func, GrName, Loc, timetrap_timeout, T, @@ -3977,8 +3982,8 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, timetrap_timeout, T, "<td>~ts</td></tr>\n", [T/1000,Comment]), FormatLoc = test_server_sup:format_loc(Loc), - print(minor, "=== location ~ts", [FormatLoc]), - print(minor, "=== reason = timetrap timeout", []), + print(minor, "=== Location: ~ts", [FormatLoc]), + print(minor, "=== Reason: timetrap timeout", []), failed; progress(failed, CaseNum, Mod, Func, GrName, Loc, {testcase_aborted,Reason}, _T, @@ -4003,13 +4008,16 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, {testcase_aborted,Reason}, _T, "<td>~ts</td></tr>\n", [Comment]), FormatLoc = test_server_sup:format_loc(Loc), - print(minor, "=== location ~ts", [FormatLoc]), - print(minor, "=== reason = {testcase_aborted,~p}", [Reason]), + print(minor, "=== Location: ~ts", [FormatLoc]), + print(minor, + escape_chars(io_lib:format("=== Reason: {testcase_aborted,~p}", + [Reason])), + []), failed; progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, Comment0, {St0,St1}) -> - print(major, "=result failed: ~p, ~w", [Reason,unknown]), + print(major, "=result failed: ~p, ~w", [Reason,unknown_location]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, @@ -4017,7 +4025,7 @@ progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; true -> "~w" end, [Time]), - ErrorReason = lists:flatten(io_lib:format("~p", [Reason])), + ErrorReason = escape_chars(lists:flatten(io_lib:format("~p", [Reason]))), ErrorReason1 = lists:flatten([string:strip(S,left) || S <- string:tokens(ErrorReason,[$\n])]), ErrorReason2 = @@ -4038,14 +4046,23 @@ progress(failed, CaseNum, Mod, Func, GrName, unknown, Reason, Time, "<td><font color=\"red\">FAILED</font></td>" "<td>~ts</td></tr>\n", [TimeStr,Comment]), - print(minor, "=== location ~w", [unknown]), + print(minor, "=== Location: ~w", [unknown]), {FStr,FormattedReason} = format_exception(Reason), - print(minor, "=== reason = " ++ FStr, [FormattedReason]), + print(minor, + escape_chars(io_lib:format("=== Reason: " ++ FStr, [FormattedReason])), + []), failed; progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, Comment0, {St0,St1}) -> - print(major, "=result failed: ~p, ~p", [Reason,Loc]), + {LocMaj,LocMin} = if Func == error_in_suite -> + case get_fw_mod(undefined) of + Mod -> {unknown_location,unknown}; + _ -> {Loc,Loc} + end; + true -> {Loc,Loc} + end, + print(major, "=result failed: ~p, ~p", [Reason,LocMaj]), print(1, "*** FAILED ~ts ***", [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]), test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName}, @@ -4058,16 +4075,17 @@ progress(failed, CaseNum, Mod, Func, GrName, Loc, Reason, Time, "" -> ""; _ -> xhtml("<br>","<br />") ++ to_string(Comment0) end, - FormatLastLoc = test_server_sup:format_loc(get_last_loc(Loc)), + FormatLastLoc = test_server_sup:format_loc(get_last_loc(LocMaj)), print(html, "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>" "<td><font color=\"red\">FAILED</font></td>" "<td><font color=\"red\">~ts</font>~ts</td></tr>\n", [TimeStr,FormatLastLoc,Comment]), - FormatLoc = test_server_sup:format_loc(Loc), - print(minor, "=== location ~ts", [FormatLoc]), + FormatLoc = test_server_sup:format_loc(LocMin), + print(minor, "=== Location: ~ts", [FormatLoc]), {FStr,FormattedReason} = format_exception(Reason), - print(minor, "=== reason = " ++ FStr, [FormattedReason]), + print(minor, "=== Reason: " ++ + escape_chars(io_lib:format(FStr, [FormattedReason])), []), failed; progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, @@ -4096,11 +4114,36 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time, "<td><font color=\"green\">Ok</font></td>" "~ts</tr>\n", [Time,Comment]), - print(minor, "=== returned value = ~p", [RetVal]), + print(minor, + escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])), + []), ok. %%-------------------------------------------------------------------- %% various help functions +escape_chars(Term) when not is_list(Term), not is_binary(Term) -> + esc_chars_in_list(io_lib:format("~tp", [Term])); +escape_chars(List = [Term | _]) when not is_list(Term), not is_integer(Term) -> + esc_chars_in_list(io_lib:format("~tp", [List])); +escape_chars(List) -> + esc_chars_in_list(List). + +esc_chars_in_list([Bin | Io]) when is_binary(Bin) -> + [Bin | esc_chars_in_list(Io)]; +esc_chars_in_list([List | Io]) when is_list(List) -> + [esc_chars_in_list(List) | esc_chars_in_list(Io)]; +esc_chars_in_list([$< | Io]) -> + ["<" | esc_chars_in_list(Io)]; +esc_chars_in_list([$> | Io]) -> + [">" | esc_chars_in_list(Io)]; +esc_chars_in_list([$& | Io]) -> + ["&" | esc_chars_in_list(Io)]; +esc_chars_in_list([Char | Io]) when is_integer(Char) -> + [Char | esc_chars_in_list(Io)]; +esc_chars_in_list([]) -> + []; +esc_chars_in_list(Bin) -> + Bin. get_fw_mod(Mod) -> case get(test_server_framework) of @@ -4314,6 +4357,10 @@ print(Detail, Format) -> print(Detail, Format, Args) -> print(Detail, Format, Args, internal). +print(Detail, ["$tc_html",Format], Args, Printer) -> + Msg = io_lib:format(Format, Args), + print_or_buffer(Detail, ["$tc_html",Msg], Printer); + print(Detail, Format, Args, Printer) -> Msg = io_lib:format(Format, Args), print_or_buffer(Detail, Msg, Printer). @@ -4684,10 +4731,10 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) -> collect_cases({Module, Cases}, St, Mode) when is_list(Cases) -> case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of - {ok, NewCases, NewSt} -> - {ok, NewCases, NewSt}; + Result = {ok,_,_} -> + Result; Other -> - {error, Other} + {error,Other} end; collect_cases({_Mod,_Case}=Spec, St, Mode) -> @@ -4705,9 +4752,9 @@ collect_case({Mod,{conf,_,_,_,_}=Conf}, St, Mode) -> collect_case(MFA, St, Mode) -> case in_skip_list(MFA, St#cc.skip) of - {true,Comment} -> + {true,Comment} when Comment /= make_failed -> {ok,[{skip_case,{MFA,Comment},Mode}],St}; - false -> + _ -> case MFA of {Mod,Case} -> collect_case_invoke(Mod, Case, MFA, St, Mode); {_Mod,_Case,_Args} -> {ok,[MFA],St} @@ -4769,17 +4816,25 @@ collect_case_subcases(Mod, Case, SubCases, St0, Mode) -> collect_files(Dir, Pattern, St, Mode) -> {ok,Cwd} = file:get_cwd(), Dir1 = filename:join(Cwd, Dir), - Wc = filename:join([Dir1,Pattern++code:objfile_extension()]), + Wc = filename:join([Dir1,Pattern++"{.erl,"++code:objfile_extension()++"}"]), case catch filelib:wildcard(Wc) of {'EXIT', Reason} -> io:format("Could not collect files: ~p~n", [Reason]), {error,{collect_fail,Dir,Pattern}}; - Mods0 -> - Mods = [{path_to_module(Mod),all} || Mod <- lists:sort(Mods0)], - collect_cases(Mods, St, Mode) + Files -> + %% convert to module names and remove duplicates + Mods = lists:foldl(fun(File, Acc) -> + Mod = fullname_to_mod(File), + case lists:member(Mod, Acc) of + true -> Acc; + false -> [Mod | Acc] + end + end, [], Files), + Tests = [{Mod,all} || Mod <- lists:sort(Mods)], + collect_cases(Tests, St, Mode) end. -path_to_module(Path) when is_list(Path) -> +fullname_to_mod(Path) when is_list(Path) -> %% If this is called with a binary, then we are probably in +fnu %% mode and have found a beam file with name encoded as latin1. We %% will let this crash since it can not work to load such a module @@ -5548,8 +5603,9 @@ html_header(Title) -> "<html>\n" "<head>\n" "<title>", Title, "</title>\n" - "<meta http-equiv=\"cache-control\" content=\"no-cache\">\n" - "<meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\n" + "<meta http-equiv=\"cache-control\" content=\"no-cache\"></meta>\n" + "<meta http-equiv=\"content-type\" content=\"text/html; " + "charset=utf-8\"></meta>\n" "</head>\n" "<body bgcolor=\"white\" text=\"black\" " "link=\"blue\" vlink=\"purple\" alink=\"red\">\n"]. diff --git a/lib/test_server/src/test_server_gl.erl b/lib/test_server/src/test_server_gl.erl index 1f7317c51d..444cfa595f 100644 --- a/lib/test_server/src/test_server_gl.erl +++ b/lib/test_server/src/test_server_gl.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -36,7 +37,8 @@ reject_io :: boolean(), %Reject I/O requests... permit_io, %... and exceptions auto_nl=true :: boolean(), %Automatically add NL - levels %{Stdout,Major,Minor} + levels, %{Stdout,Major,Minor} + escape_chars=true %Switch escaping HTML on/off }). %% start_link() @@ -129,6 +131,10 @@ set_props(GL, PropList) -> %%% Internal functions. init([]) -> + EscChars = case application:get_env(test_server, esc_chars) of + {ok,ECBool} -> ECBool; + _ -> true + end, {ok,#st{tc_supervisor=none, minor=none, minor_monitor=none, @@ -136,7 +142,8 @@ init([]) -> reject_io=false, permit_io=gb_sets:empty(), auto_nl=true, - levels={1,19,10} + levels={1,19,10}, + escape_chars=EscChars }}. req(GL, Req) -> @@ -181,7 +188,7 @@ handle_info({io_request,From,ReplyAs,Req}=IoReq, St) -> try io_req(Req, From, St) of passthrough -> group_leader() ! IoReq; - Data -> + {EscapeHtml,Data} -> case is_io_permitted(From, St) of false -> ok; @@ -192,7 +199,13 @@ handle_info({io_request,From,ReplyAs,Req}=IoReq, St) -> #st{capture=CapturePid} -> CapturePid ! {captured,Data} end, - output(minor, Data, From, From, St) + case EscapeHtml andalso St#st.escape_chars of + true -> + output(minor, test_server_ctrl:escape_chars(Data), + From, From, St); + false -> + output(minor, Data, From, From, St) + end end, From ! {io_reply,ReplyAs,ok} catch @@ -203,9 +216,20 @@ handle_info({io_request,From,ReplyAs,Req}=IoReq, St) -> handle_info({structured_io,ClientPid,{Detail,Str}}, St) -> output(Detail, Str, ClientPid, ClientPid, St), {noreply,St}; +handle_info({printout,Detail,["$tc_html",Format],Args}, St) -> + Str = io_lib:format(Format, Args), + output(Detail, ["$tc_html",Str], internal, none, St), + {noreply,St}; +handle_info({printout,Detail,Fun}, St) when is_function(Fun)-> + output(Detail, Fun, internal, none, St), + {noreply,St}; handle_info({printout,Detail,Format,Args}, St) -> Str = io_lib:format(Format, Args), - output(Detail, Str, internal, none, St), + if not St#st.escape_chars -> + output(Detail, ["$tc_html",Str], internal, none, St); + true -> + output(Detail, Str, internal, none, St) + end, {noreply,St}; handle_info(Msg, #st{tc_supervisor=Pid}=St) when is_pid(Pid) -> %% The process overseeing the testcase process also used to be @@ -230,25 +254,55 @@ do_set_props([{reject_io_reqs,Bool}|Ps], St) -> do_set_props(Ps, St#st{reject_io=Bool}); do_set_props([], St) -> St. -io_req({put_chars,Enc,Bytes}, _, _) when Enc =:= latin1; Enc =:= unicode -> - unicode:characters_to_list(Bytes, Enc); +io_req({put_chars,Enc,Str}, _, _) when Enc =:= latin1; Enc =:= unicode -> + case Str of + ["$tc_html",Str0] -> + {false,unicode:characters_to_list(Str0, Enc)}; + _ -> + {true,unicode:characters_to_list(Str, Enc)} + end; io_req({put_chars,Encoding,Mod,Func,[Format,Args]}, _, _) -> - Str = Mod:Func(Format, Args), - unicode:characters_to_list(Str, Encoding); + case Format of + ["$tc_html",Format0] -> + Str = Mod:Func(Format0, Args), + {false,unicode:characters_to_list(Str, Encoding)}; + _ -> + Str = Mod:Func(Format, Args), + {true,unicode:characters_to_list(Str, Encoding)} + end; io_req(_, _, _) -> passthrough. -output(Level, Str, Sender, From, St) when is_integer(Level) -> +output(Level, StrOrFun, Sender, From, St) when is_integer(Level) -> case selected_by_level(Level, stdout, St) of - true -> output(stdout, Str, Sender, From, St); - false -> ok + true when hd(StrOrFun) == "$tc_html" -> + output(stdout, tl(StrOrFun), Sender, From, St); + true when is_function(StrOrFun) -> + output(stdout, StrOrFun(stdout), Sender, From, St); + true -> + output(stdout, StrOrFun, Sender, From, St); + false -> + ok end, case selected_by_level(Level, major, St) of - true -> output(major, Str, Sender, From, St); - false -> ok + true when hd(StrOrFun) == "$tc_html" -> + output(major, tl(StrOrFun), Sender, From, St); + true when is_function(StrOrFun) -> + output(major, StrOrFun(major), Sender, From, St); + true -> + output(major, StrOrFun, Sender, From, St); + false -> + ok end, case selected_by_level(Level, minor, St) of - true -> output(minor, Str, Sender, From, St); - false -> ok + true when hd(StrOrFun) == "$tc_html" -> + output(minor, tl(StrOrFun), Sender, From, St); + true when is_function(StrOrFun) -> + output(minor, StrOrFun(minor), Sender, From, St); + true -> + output(minor, test_server_ctrl:escape_chars(StrOrFun), + Sender, From, St); + false -> + ok end; output(stdout, Str, _Sender, From, St) -> output_to_file(stdout, Str, From, St); diff --git a/lib/test_server/src/test_server_internal.hrl b/lib/test_server/src/test_server_internal.hrl index bafeeaabfe..1ec2d83417 100644 --- a/lib/test_server/src/test_server_internal.hrl +++ b/lib/test_server/src/test_server_internal.hrl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2002-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -29,7 +30,6 @@ version, % string() system_version, % string() root_dir, % string() - test_server_dir, % string() emulator, % string() otp_release, % string() username, % string() diff --git a/lib/test_server/src/test_server_io.erl b/lib/test_server/src/test_server_io.erl index 62af3d5b28..0d881d0ada 100644 --- a/lib/test_server/src/test_server_io.erl +++ b/lib/test_server/src/test_server_io.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/test_server_node.erl b/lib/test_server/src/test_server_node.erl index acd47788db..37f8941d24 100644 --- a/lib/test_server/src/test_server_node.erl +++ b/lib/test_server/src/test_server_node.erl @@ -1,18 +1,19 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2014. All Rights Reserved. +%% Copyright Ericsson AB 2002-2016. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -120,7 +121,7 @@ start_tracer_node(TraceFile,TI) -> %%% trace_nodes(Sock,Nodes) -> Bin = term_to_binary({add_nodes,Nodes}), - ok = gen_tcp:send(Sock, [1|Bin]), + ok = gen_tcp:send(Sock, tag_trace_message(Bin)), receive_ack(Sock). @@ -141,7 +142,7 @@ receive_ack(Sock) -> %%% stop_tracer_node(Sock) -> Bin = term_to_binary(id(stop)), - ok = gen_tcp:send(Sock, [1|Bin]), + ok = gen_tcp:send(Sock, tag_trace_message(Bin)), receive {tcp_closed,Sock} -> gen_tcp:close(Sock) end, ok. @@ -170,7 +171,7 @@ trc([TraceFile, PortAtom, Type]) -> {packet,2}]) of {ok,Sock} -> BinResult = term_to_binary(Result), - ok = gen_tcp:send(Sock,[1|BinResult]), + ok = gen_tcp:send(Sock,tag_trace_message(BinResult)), trc_loop(Sock,Patterns,Type); _else -> ok @@ -186,7 +187,7 @@ trc_loop(Sock,Patterns,Type) -> {ok,{add_nodes,Nodes}} -> add_nodes(Nodes,Patterns,Type), Bin = term_to_binary(id(ok)), - ok = gen_tcp:send(Sock, [1|Bin]), + ok = gen_tcp:send(Sock, tag_trace_message(Bin)), trc_loop(Sock,Patterns,Type); {ok,stop} -> ttb:stop(), @@ -306,11 +307,11 @@ start_node_peer(SlaveName, OptList, From, TI) -> HostStr, " ", WaitPort]), % Support for erl_crash_dump files.. - CrashFile = filename:join([TI#target_info.test_server_dir, + CrashDir = test_server_sup:crash_dump_dir(), + CrashFile = filename:join([CrashDir, "erl_crash_dump."++cast_to_list(SlaveName)]), CrashArgs = lists:concat([" -env ERL_CRASH_DUMP \"",CrashFile,"\" "]), FailOnError = start_node_get_option_value(fail_on_error, OptList, true), - Pa = TI#target_info.test_server_dir, Prog0 = start_node_get_option_value(erl, OptList, default), Prog = quote_progname(pick_erl_program(Prog0)), Args = @@ -321,7 +322,6 @@ start_node_peer(SlaveName, OptList, From, TI) -> Cmd = lists:concat([Prog, " -detached ", TI#target_info.naming, " ", SlaveName, - " -pa \"", Pa,"\"", NodeStarted, CrashArgs, " ", Args]), @@ -353,28 +353,31 @@ start_node_peer(SlaveName, OptList, From, TI) -> I = "=== Not waiting for node", gen_server:reply(From,{{ok, Nodename}, HostStr, Cmd, I, []}), Self = self(), - spawn_link( - fun() -> - wait_for_node_started(LSock,Tmo,undefined, - Cleanup,TI,Self), - receive after infinity -> ok end - end), + spawn_link(wait_for_node_started_fun(LSock,Tmo,Cleanup,TI,Self)), ok end. +-spec wait_for_node_started_fun(_, _, _, _, _) -> fun(() -> no_return()). +wait_for_node_started_fun(LSock, Tmo, Cleanup, TI, Self) -> + fun() -> + wait_for_node_started(LSock,Tmo,undefined, + Cleanup,TI,Self), + receive after infinity -> ok end + end. + %% %% Slave nodes are started on a remote host if %% - the option remote is given when calling test_server:start_node/3 %% -start_node_slave(SlaveName, OptList, From, TI) -> +start_node_slave(SlaveName, OptList, From, _TI) -> SuppliedArgs = start_node_get_option_value(args, OptList, []), Cleanup = start_node_get_option_value(cleanup, OptList, true), - CrashFile = filename:join([TI#target_info.test_server_dir, + CrashDir = test_server_sup:crash_dump_dir(), + CrashFile = filename:join([CrashDir, "erl_crash_dump."++cast_to_list(SlaveName)]), CrashArgs = lists:concat([" -env ERL_CRASH_DUMP \"",CrashFile,"\" "]), - Pa = TI#target_info.test_server_dir, - Args = lists:concat([" -pa \"", Pa, "\" ", SuppliedArgs, CrashArgs]), + Args = lists:concat([" ", SuppliedArgs, CrashArgs]), Prog0 = start_node_get_option_value(erl, OptList, default), Prog = pick_erl_program(Prog0), @@ -467,7 +470,11 @@ handle_start_node_return(Version,VsnStr,{started, Node, OVersion, OVsnStr}) -> node_started([Host,PortAtom]) -> %% Must spawn a new process because the boot process should not %% hang forever!! - spawn(fun() -> node_started(Host,PortAtom) end). + spawn(node_started_fun(Host,PortAtom)). + +-spec node_started_fun(_, _) -> fun(() -> no_return()). +node_started_fun(Host,PortAtom) -> + fun() -> node_started(Host,PortAtom) end. %% This process hangs forever, just waiting for the socket to be %% closed and terminating the node @@ -481,7 +488,7 @@ node_started(Host,PortAtom) -> {ok,Sock} -> Started = term_to_binary({started, node(), Version, VsnStr}), - ok = gen_tcp:send(Sock, [1|Started]), + ok = gen_tcp:send(Sock, tag_trace_message(Started)), receive _Anyting -> gen_tcp:close(Sock), erlang:halt() @@ -491,8 +498,10 @@ node_started(Host,PortAtom) -> end. - - +-compile({inline, [tag_trace_message/1]}). +-dialyzer({no_improper_lists, tag_trace_message/1}). +tag_trace_message(M) -> + [1|M]. % start_which_node(Optlist) -> hostname start_which_node(Optlist) -> @@ -618,9 +627,8 @@ do_quote_progname([Prog,Arg|Args]) -> end. random_element(L) -> - {A,B,C} = now(), - E = lists:sum([A,B,C]) rem length(L), - lists:nth(E+1, L). + random:seed(os:timestamp()), + lists:nth(random:uniform(length(L)), L). find_release(latest) -> "/usr/local/otp/releases/latest/bin/erl"; diff --git a/lib/test_server/src/test_server_sup.erl b/lib/test_server/src/test_server_sup.erl index 15a6fdd1de..c4530ba62f 100644 --- a/lib/test_server/src/test_server_sup.erl +++ b/lib/test_server/src/test_server_sup.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1998-2014. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -125,14 +126,8 @@ messages_get(Msgs) -> end. timecall(M, F, A) -> - Befor = erlang:now(), - Val = apply(M, F, A), - After = erlang:now(), - Elapsed = - (element(1,After)*1000000+element(2,After)+element(3,After)/1000000)- - (element(1,Befor)*1000000+element(2,Befor)+element(3,Befor)/1000000), - {Elapsed, Val}. - + {Elapsed, Val} = timer:tc(M, F, A), + {Elapsed / 1000000, Val}. call_crash(Time,Crash,M,F,A) -> @@ -599,7 +594,11 @@ cleanup_crash_dumps() -> delete_files(Dumps). crash_dump_dir() -> - filename:dirname(code:which(?MODULE)). + %% If no framework is known, then we use current working directory + %% - in most cases that will be the same as the default log + %% directory. + {ok,Dir} = test_server_sup:framework_call(get_log_dir,[],file:get_cwd()), + Dir. tar_crash_dumps() -> Dir = crash_dump_dir(), @@ -887,9 +886,8 @@ unique_name() -> util_loop(State) -> receive {From,unique_name} -> - {_,S,Us} = now(), - Ms = trunc(Us/1000), - Name = lists:flatten(io_lib:format("~w.~w", [S,Ms])), + Nr = erlang:unique_integer([positive]), + Name = integer_to_list(Nr), if Name == State#util_state.latest_name -> timer:sleep(1), self() ! {From,unique_name}, diff --git a/lib/test_server/src/things/distr_startup_SUITE.erl b/lib/test_server/src/things/distr_startup_SUITE.erl index 0d4e467570..aa84ab007f 100644 --- a/lib/test_server/src/things/distr_startup_SUITE.erl +++ b/lib/test_server/src/things/distr_startup_SUITE.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1996-2009. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/things/mnesia_power_SUITE.erl b/lib/test_server/src/things/mnesia_power_SUITE.erl index 281dac7742..e9bc75e583 100644 --- a/lib/test_server/src/things/mnesia_power_SUITE.erl +++ b/lib/test_server/src/things/mnesia_power_SUITE.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1996-2009. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/things/random_kill_SUITE.erl b/lib/test_server/src/things/random_kill_SUITE.erl index 4fcd63e3af..917bc2b3d5 100644 --- a/lib/test_server/src/things/random_kill_SUITE.erl +++ b/lib/test_server/src/things/random_kill_SUITE.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1996-2009. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/things/verify.erl b/lib/test_server/src/things/verify.erl index eac20c013e..b09d0fbda9 100644 --- a/lib/test_server/src/things/verify.erl +++ b/lib/test_server/src/things/verify.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1996-2009. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index d6d2e865e2..8bbdc8f8cf 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -24,15 +25,20 @@ -module(ts). --export([run/0, run/1, run/2, run/3, run/4, run/5, - tests/0, tests/1, +-export([cl_run/1, + run/0, run/1, run/2, run/3, run/4, run/5, + run_category/1, run_category/2, run_category/3, + tests/0, tests/1, suites/1, categories/1, install/0, install/1, - bench/0, bench/1, bench/2, benchmarks/0, - smoke_test/0, smoke_test/1,smoke_test/2, smoke_tests/0, estone/0, estone/1, cross_cover_analyse/1, compile_testcases/0, compile_testcases/1, help/0]). + +%% Functions kept for backwards compatibility +-export([bench/0, bench/1, bench/2, benchmarks/0, + smoke_test/0, smoke_test/1,smoke_test/2, smoke_tests/0]). + -export([i/0, l/1, r/0, r/1, r/2, r/3]). %%%---------------------------------------------------------------------- @@ -82,10 +88,13 @@ -define( install_help, [ - " ts:install() - Install TS with no Options.\n" - " ts:install([Options]) - Install TS with Options\n" + " ts:install()\n", + " Install ts with no options.\n", "\n", - "Installation options supported:\n", + " ts:install(Options)\n", + " Install ts with a list of options, see below.\n", + "\n", + "Installation options supported:\n\n", " {longnames, true} - Use fully qualified hostnames\n", " {verbose, Level} - Sets verbosity level for TS output (0,1,2), 0 is\n" " quiet(default).\n" @@ -110,21 +119,64 @@ help() -> end. help(uninstalled) -> - H = ["TS is not installed yet. To install use:\n\n"], + H = ["ts is not yet installed. To install use:\n\n"], show_help([H,?install_help]); help(installed) -> - H = ["Run functions:\n", - " ts:run() - Run all available tests.\n", - " ts:run(Spec) - Run all tests in given test spec file.\n", - " The spec file is actually ../*_test/Spec.spec\n", - " ts:run([Specs]) - Run all tests in all given test spec files.\n", - " ts:run(Spec, Mod) - Run a single test suite.\n", - " ts:run(Spec, Mod, Case)\n", - " - Run a single test case.\n", - " All above run functions can have an additional Options argument\n", - " which is a list of options.\n", + H = ["\n", + "Run functions:\n\n", + " ts:run()\n", + " Run the tests for all apps. The tests are defined by the\n", + " main test specification for each app: ../App_test/App.spec.\n", + "\n", + " ts:run(Apps)\n", + " Apps = atom() | [atom()]\n", + " Run the tests for an app, or set of apps. The tests are\n", + " defined by the main test specification for each app:\n", + " ../App_test/App.spec.\n", + "\n", + " ts:run(App, Suites)\n", + " App = atom(), Suites = atom() | [atom()]\n", + " Run one or more test suites for App (i.e. modules named\n", + " *_SUITE.erl, located in ../App_test/).\n", + "\n", + " ts:run(App, Suite, TestCases)\n", + " App = atom(), Suite = atom(),\n", + " TestCases = TCs | {testcase,TCs}, TCs = atom() | [atom()]\n", + " Run one or more test cases (functions) in Suite.\n", + "\n", + " ts:run(App, Suite, {group,Groups})\n", + " App = atom(), Suite = atom(), Groups = atom() | [atom()]\n", + " Run one or more test case groups in Suite.\n", + "\n", + " ts:run(App, Suite, {group,Group}, {testcase,TestCases})\n", + " App = atom(), Suite = atom(), Group = atom(),\n", + " TestCases = atom() | [atom()]\n", + " Run one or more test cases in a test case group in Suite.\n", + "\n", + " ts:run_category(TestCategory)\n", + " TestCategory = smoke | essential | bench | atom()\n", + " Run the specified category of tests for all apps.\n", + " For each app, the tests are defined by the specification:\n", + " ../App_test/App_TestCategory.spec.\n", + "\n", + " ts:run_category(Apps, TestCategory)\n", + " Apps = atom() | [atom()],\n", + " TestCategory = smoke | essential | bench | atom()\n", + " Run the specified category of tests for the given app or apps.\n", "\n", - "Run options supported:\n", + " Note that the test category parameter may have arbitrary value,\n", + " but should correspond to an existing test specification with file\n", + " name: ../App_test/App_TestCategory.spec.\n", + " Predefined categories exist for smoke tests, essential tests and\n", + " benchmark tests. The corresponding specs are:\n", + " ../*_test/Spec_smoke.spec, ../*_test/Spec_essential.spec and\n", + " ../*_test/Spec_bench.spec.\n", + "\n", + " All above run functions can take an additional last argument,\n", + " Options, which is a list of options (e.g. ts:run(App, Options),\n", + " or ts:run_category(Apps, TestCategory, Options)).\n", + "\n", + "Run options supported:\n\n", " batch - Do not start a new xterm\n", " {verbose, Level} - Same as the verbosity option for install\n", " verbose - Same as {verbose, 1}\n", @@ -143,47 +195,46 @@ help(installed) -> " files are. The default location is\n" " tests/test_server/.\n" "\n", - "Supported trace information elements\n", + "Supported trace information elements:\n\n", " {tp | tpl, Mod, [] | match_spec()}\n", " {tp | tpl, Mod, Func, [] | match_spec()}\n", " {tp | tpl, Mod, Func, Arity, [] | match_spec()}\n", " {ctp | ctpl, Mod}\n", " {ctp | ctpl, Mod, Func}\n", " {ctp | ctpl, Mod, Func, Arity}\n", + "\n\n", + "Support functions:\n\n", + " ts:tests()\n", + " Returns all apps available for testing.\n", + "\n", + " ts:tests(TestCategory)\n", + " Returns all apps that provide tests in the given category.\n", + "\n", + " ts:suites(App)\n", + " Returns all available test suites for App,\n", + " i.e. ../App_test/*_SUITE.erl\n", + "\n", + " ts:categories(App)\n", + " Returns all test categories available for App.\n", "\n", - "Support functions:\n", - " ts:tests() - Shows all available families of tests.\n", - " ts:tests(Spec) - Shows all available test modules in Spec,\n", - " i.e. ../Spec_test/*_SUITE.erl\n", - " ts:estone() - Run estone_SUITE in kernel application with\n" - " no run options\n", - " ts:estone(Opts) - Run estone_SUITE in kernel application with\n" - " the given run options\n", - " ts:cross_cover_analyse(Level)\n" - " - Used after ts:run with option cover or \n" - " cover_details. Analyses modules specified with\n" - " a 'cross' statement in the cover spec file.\n" - " Level can be 'overview' or 'details'.\n", - " ts:compile_testcases()~n" - " ts:compile_testcases(Apps)~n" - " - Compile all testcases for usage in a cross ~n" - " compile environment." - " \n" - "Benchmark functions:\n" - " ts:benchmarks() - Get all available families of benchmarks\n" - " ts:bench() - Runs all benchmarks\n" - " ts:bench(Spec) - Runs all benchmarks in the given spec file.\n" - " The spec file is actually ../*_test/Spec_bench.spec\n\n" - " ts:bench can take the same Options argument as ts:run.\n" - "Smoke test functions:\n" - " ts:smoke_tests() - Get all available families of smoke tests\n" - " ts:smoke_test() - Runs all smoke tests\n" - " ts:smoke_test(Spec)\n" - " - Runs all smoke tests in the given spec file.\n" - " The spec file is actually ../*_test/Spec_smoke.spec\n\n" - " ts:smoke_test can take the same Options argument as ts:run.\n" - "\n" - "Installation (already done):\n" + " ts:estone()\n", + " Runs estone_SUITE in the kernel application with no run options\n", + "\n", + " ts:estone(Opts)\n", + " Runs estone_SUITE in the kernel application with the given\n", + " run options\n", + "\n", + " ts:cross_cover_analyse(Level)\n", + " Use after ts:run with option cover or cover_details. Analyses\n", + " modules specified with a 'cross' statement in the cover spec file.\n", + " Level can be 'overview' or 'details'.\n", + "\n", + " ts:compile_testcases()\n", + " ts:compile_testcases(Apps)\n", + " Compiles all test cases for the given apps, for usage in a\n", + " cross compilation environment.\n", + "\n\n", + "Installation (already done):\n\n" ], show_help([H,?install_help]). @@ -212,86 +263,138 @@ run_all(_Vars) -> run_some([], _Opts) -> ok; -run_some([{Spec,Mod}|Specs], Opts) -> - case run(Spec, Mod, Opts) of +run_some(Apps, Opts) -> + case proplists:get_value(test_category, Opts) of + bench -> + check_and_run(fun(Vars) -> ts_benchmark:run(Apps, Opts, Vars) end); + _Other -> + run_some1(Apps, Opts) + end. + +run_some1([], _Opts) -> + ok; +run_some1([{App,Mod}|Apps], Opts) -> + case run(App, Mod, Opts) of ok -> ok; - Error -> io:format("~p: ~p~n",[{Spec,Mod},Error]) + Error -> io:format("~p: ~p~n",[{App,Mod},Error]) end, - run_some(Specs, Opts); -run_some([Spec|Specs], Opts) -> - case run(Spec, Opts) of + run_some1(Apps, Opts); +run_some1([App|Apps], Opts) -> + case run(App, Opts) of ok -> ok; - Error -> io:format("~p: ~p~n",[Spec,Error]) + Error -> io:format("~p: ~p~n",[App,Error]) end, - run_some(Specs, Opts). - -%% Runs one test spec (interactive). -run(Testspec) when is_atom(Testspec) -> - Options=check_test_get_opts(Testspec, []), - File = atom_to_list(Testspec), - run_test(File, [{spec,[File++".spec"]}], Options); + run_some1(Apps, Opts). + +%% This can be used from command line. Both App and +%% TestCategory must be specified. App may be 'all' +%% and TestCategory may be 'main'. Examples: +%% erl -s ts cl_run kernel smoke <options> +%% erl -s ts cl_run kernel main <options> +%% erl -s ts cl_run all essential <options> +%% erl -s ts cl_run all main <options> +%% When using the 'main' category and running with cover, +%% one can also use the cross_cover_analysis flag. +cl_run([App,Cat|Options0]) when is_atom(App) -> -%% This can be used from command line, e.g. -%% erl -s ts run all_tests <config> -%% When using the all_tests flag and running with cover, one can also -%% use the cross_cover_analysis flag. -run([all_tests|Config0]) -> AllAtomsFun = fun(X) when is_atom(X) -> true; (_) -> false end, - Config1 = - case lists:all(AllAtomsFun,Config0) of + Options1 = + case lists:all(AllAtomsFun, Options0) of true -> %% Could be from command line - lists:map(fun(Conf)->to_erlang_term(Conf) end,Config0)--[batch]; + lists:map(fun(Opt) -> + to_erlang_term(Opt) + end, Options0) -- [batch]; false -> - Config0--[batch] + Options0 -- [batch] end, %% Make sure there is exactly one occurence of 'batch' - Config2 = [batch|Config1], - - R = run(tests(),Config2), - - case check_for_cross_cover_analysis_flag(Config2) of + Options2 = [batch|Options1], + + Result = + case {App,Cat} of + {all,main} -> + run(tests(), Options2); + {all,Cat} -> + run_category(Cat, Options2); + {_,main} -> + run(App, Options2); + {_,Cat} -> + run_category(App, Cat, Options2) + end, + case check_for_cross_cover_analysis_flag(Options2) of false -> ok; Level -> cross_cover_analyse(Level) end, + Result. - R; +%% run/1 +%% Runs tests for one app (interactive). +run(App) when is_atom(App) -> + Options = check_test_get_opts(App, []), + File = atom_to_list(App), + run_test(File, [{spec,[File++".spec"]},{allow_user_terms,true}], Options); -%% ts:run(ListOfTests) -run(List) when is_list(List) -> - run(List, [batch]). - -run(List, Opts) when is_list(List), is_list(Opts) -> - run_some(List, Opts); +%% This can be used from command line, e.g. +%% erl -s ts run all <options> +%% erl -s ts run main <options> +run([all,main|Opts]) -> + cl_run([all,main|Opts]); +run([all|Opts]) -> + cl_run([all,main|Opts]); +run([main|Opts]) -> + cl_run([all,main|Opts]); +%% Backwards compatible +run([all_tests|Opts]) -> + cl_run([all,main|Opts]); + +%% run/1 +%% Runs the main tests for all available apps +run(Apps) when is_list(Apps) -> + run(Apps, [batch]). %% run/2 -%% Runs one test spec with list of suites or with options -run(Testspec, ModsOrConfig) when is_atom(Testspec), - is_list(ModsOrConfig) -> - case is_list_of_suites(ModsOrConfig) of +%% Runs the main tests for all available apps +run(Apps, Opts) when is_list(Apps), is_list(Opts) -> + run_some(Apps, Opts); + +%% Runs tests for one app with list of suites or with options +run(App, ModsOrOpts) when is_atom(App), + is_list(ModsOrOpts) -> + case is_list_of_suites(ModsOrOpts) of false -> - run(Testspec, {config_list,ModsOrConfig}); + run(App, {opts_list,ModsOrOpts}); true -> - run_some([{Testspec,M} || M <- ModsOrConfig], + run_some([{App,M} || M <- ModsOrOpts], [batch]) end; -run(Testspec, {config_list,Config}) -> - Options=check_test_get_opts(Testspec, Config), - IsSmoke=proplists:get_value(smoke,Config), - File=atom_to_list(Testspec), + +run(App, {opts_list,Opts}) -> + Options = check_test_get_opts(App, Opts), + File = atom_to_list(App), + + %% check if other test category than main has been specified + {CatSpecName,TestCat} = + case proplists:get_value(test_category, Opts) of + undefined -> + {"",main}; + Cat -> + {"_" ++ atom_to_list(Cat),Cat} + end, + WhatToDo = - case Testspec of + case App of %% Known to exist but fails generic tests below emulator -> test; system -> test; erl_interface -> test; epmd -> test; _ -> - case code:lib_dir(Testspec) of + case code:lib_dir(App) of {error,bad_name} -> %% Application does not exist skip; @@ -313,92 +416,167 @@ run(Testspec, {config_list,Config}) -> end end end, - Spec = - case WhatToDo of - skip -> - create_skip_spec(Testspec, tests(Testspec)); - test when IsSmoke -> - File++"_smoke.spec"; - test -> - File++".spec" - end, - run_test(File, [{spec,[Spec]}], Options); -%% Runs one module in a spec (interactive) -run(Testspec, Mod) when is_atom(Testspec), is_atom(Mod) -> - run_test({atom_to_list(Testspec),Mod}, + case WhatToDo of + skip -> + SkipSpec = create_skip_spec(App, suites(App)), + run_test(File, [{spec,[SkipSpec]}], Options); + test when TestCat == bench -> + check_and_run(fun(Vars) -> + ts_benchmark:run([App], Options, Vars) + end); + test -> + Spec = File ++ CatSpecName ++ ".spec", + run_test(File, [{spec,[Spec]},{allow_user_terms,true}], Options) + end; + +%% Runs one module for an app (interactive) +run(App, Mod) when is_atom(App), is_atom(Mod) -> + run_test({atom_to_list(App),Mod}, [{suite,Mod}], [interactive]). %% run/3 -%% Run one module in a spec with Config -run(Testspec, Mod, Config) when is_atom(Testspec), - is_atom(Mod), - is_list(Config) -> - Options=check_test_get_opts(Testspec, Config), - run_test({atom_to_list(Testspec),Mod}, +%% Run one module for an app with Opts +run(App, Mod, Opts) when is_atom(App), + is_atom(Mod), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), + run_test({atom_to_list(App),Mod}, [{suite,Mod}], Options); -%% Run multiple modules with Config -run(Testspec, Mods, Config) when is_atom(Testspec), - is_list(Mods), - is_list(Config) -> - run_some([{Testspec,M} || M <- Mods], Config); + +%% Run multiple modules with Opts +run(App, Mods, Opts) when is_atom(App), + is_list(Mods), + is_list(Opts) -> + run_some([{App,M} || M <- Mods], Opts); + %% Runs one test case in a module. -run(Testspec, Mod, Case) when is_atom(Testspec), - is_atom(Mod), - is_atom(Case) -> - Options=check_test_get_opts(Testspec, []), +run(App, Mod, Case) when is_atom(App), + is_atom(Mod), + is_atom(Case) -> + Options = check_test_get_opts(App, []), Args = [{suite,Mod},{testcase,Case}], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + %% Runs one or more groups in a module. -run(Testspec, Mod, Grs={group,_Groups}) when is_atom(Testspec), - is_atom(Mod) -> - Options=check_test_get_opts(Testspec, []), +run(App, Mod, Grs={group,_Groups}) when is_atom(App), + is_atom(Mod) -> + Options = check_test_get_opts(App, []), Args = [{suite,Mod},Grs], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + %% Runs one or more test cases in a module. -run(Testspec, Mod, TCs={testcase,_Cases}) when is_atom(Testspec), - is_atom(Mod) -> - Options=check_test_get_opts(Testspec, []), +run(App, Mod, TCs={testcase,_Cases}) when is_atom(App), + is_atom(Mod) -> + Options = check_test_get_opts(App, []), Args = [{suite,Mod},TCs], - run_test(atom_to_list(Testspec), Args, Options). + run_test(atom_to_list(App), Args, Options). %% run/4 %% Run one test case in a module with Options. -run(Testspec, Mod, Case, Config) when is_atom(Testspec), - is_atom(Mod), - is_atom(Case), - is_list(Config) -> - Options=check_test_get_opts(Testspec, Config), +run(App, Mod, Case, Opts) when is_atom(App), + is_atom(Mod), + is_atom(Case), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},{testcase,Case}], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + %% Run one or more test cases in a module with Options. -run(Testspec, Mod, {testcase,Cases}, Config) when is_atom(Testspec), - is_atom(Mod) -> - run(Testspec, Mod, Cases, Config); -run(Testspec, Mod, Cases, Config) when is_atom(Testspec), - is_atom(Mod), - is_list(Cases), - is_list(Config) -> - Options=check_test_get_opts(Testspec, Config), +run(App, Mod, {testcase,Cases}, Opts) when is_atom(App), + is_atom(Mod) -> + run(App, Mod, Cases, Opts); +run(App, Mod, Cases, Opts) when is_atom(App), + is_atom(Mod), + is_list(Cases), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},Cases], - run_test(atom_to_list(Testspec), Args, Options); + run_test(atom_to_list(App), Args, Options); + +%% Run one or more test cases in a group. +run(App, Mod, Gr={group,_Group}, {testcase,Cases}) when is_atom(App), + is_atom(Mod) -> + run(App, Mod, Gr, Cases, [batch]); + + %% Run one or more groups in a module with Options. -run(Testspec, Mod, Grs={group,_Groups}, Config) when is_atom(Testspec), - is_atom(Mod) -> - Options=check_test_get_opts(Testspec, Config), +run(App, Mod, Grs={group,_Groups}, Opts) when is_atom(App), + is_atom(Mod), + is_list(Opts) -> + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},Grs], - run_test(atom_to_list(Testspec), Args, Options). + run_test(atom_to_list(App), Args, Options). %% run/5 %% Run one or more test cases in a group with Options. -run(Testspec, Mod, Group, Cases, Config) when is_atom(Testspec), - is_atom(Mod), - is_list(Config) -> +run(App, Mod, Group, Cases, Opts) when is_atom(App), + is_atom(Mod), + is_list(Opts) -> Group1 = if is_tuple(Group) -> Group; true -> {group,Group} end, Cases1 = if is_tuple(Cases) -> Cases; true -> {testcase,Cases} end, - Options=check_test_get_opts(Testspec, Config), + Options = check_test_get_opts(App, Opts), Args = [{suite,Mod},Group1,Cases1], - run_test(atom_to_list(Testspec), Args, Options). + run_test(atom_to_list(App), Args, Options). + +%% run_category/1 +run_category(TestCategory) when is_atom(TestCategory) -> + run_category(TestCategory, [batch]). + +%% run_category/2 +run_category(TestCategory, Opts) when is_atom(TestCategory), + is_list(Opts) -> + case ts:tests(TestCategory) of + [] -> + {error, no_tests_available}; + Apps -> + Opts1 = [{test_category,TestCategory} | Opts], + run_some(Apps, Opts1) + end; + +run_category(Apps, TestCategory) when is_atom(TestCategory) -> + run_category(Apps, TestCategory, [batch]). + +%% run_category/3 +run_category(App, TestCategory, Opts) -> + Apps = if is_atom(App) -> [App]; + is_list(App) -> App + end, + Opts1 = [{test_category,TestCategory} | Opts], + run_some(Apps, Opts1). + +%%----------------------------------------------------------------- +%% Functions kept for backwards compatibility + +bench() -> + run_category(bench, []). +bench(Opts) when is_list(Opts) -> + run_category(bench, Opts); +bench(App) -> + run_category(App, bench, []). +bench(App, Opts) when is_atom(App) -> + run_category(App, bench, Opts); +bench(Apps, Opts) when is_list(Apps) -> + run_category(Apps, bench, Opts). + +benchmarks() -> + tests(bench). + +smoke_test() -> + run_category(smoke, []). +smoke_test(Opts) when is_list(Opts) -> + run_category(smoke, Opts); +smoke_test(App) -> + run_category(App, smoke, []). +smoke_test(App, Opts) when is_atom(App) -> + run_category(App, smoke, Opts); +smoke_test(Apps, Opts) when is_list(Apps) -> + run_category(Apps, smoke, Opts). + +smoke_tests() -> + tests(smoke). + +%%----------------------------------------------------------------- is_list_of_suites(List) -> lists:all(fun(Suite) -> @@ -416,29 +594,29 @@ is_list_of_suites(List) -> %% Create a spec to skip all SUITES, this is used when the application %% to be tested is not part of the OTP release to be tested. -create_skip_spec(Testspec, SuitesToSkip) -> +create_skip_spec(App, SuitesToSkip) -> {ok,Cwd} = file:get_cwd(), - TestspecString = atom_to_list(Testspec), - Specname = TestspecString++"_skip.spec", + AppString = atom_to_list(App), + Specname = AppString++"_skip.spec", {ok,D} = file:open(filename:join([filename:dirname(Cwd), - TestspecString++"_test",Specname]), + AppString++"_test",Specname]), [write]), - TestDir = "\"../"++TestspecString++"_test\"", + TestDir = "\"../"++AppString++"_test\"", io:format(D,"{suites, "++TestDir++", all}.~n",[]), io:format(D,"{skip_suites, "++TestDir++", ~w, \"Skipped as application" " is not in path!\"}.",[SuitesToSkip]), Specname. -%% Check testspec to be valid and get possible Options -%% from the config. -check_test_get_opts(Testspec, Config) -> - validate_test(Testspec), - Mode = configmember(batch, {batch, interactive}, Config), - Vars = configvars(Config), - Trace = get_config(trace,Config), - ConfigPath = get_config(config,Config), - KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Config), - Cover = configcover(Testspec,Config), +%% Check testspec for App to be valid and get possible options +%% from the list. +check_test_get_opts(App, Opts) -> + validate_test(App), + Mode = configmember(batch, {batch, interactive}, Opts), + Vars = configvars(Opts), + Trace = get_config(trace,Opts), + ConfigPath = get_config(config,Opts), + KeepTopcase = configmember(keep_topcase, {keep_topcase,[]}, Opts), + Cover = configcover(App,Opts), lists:flatten([Vars,Mode,Trace,KeepTopcase,Cover,ConfigPath]). to_erlang_term(Atom) -> @@ -447,7 +625,7 @@ to_erlang_term(Atom) -> {ok, Term} = erl_parse:parse_term(Tokens), Term. -%% Validate that a Testspec really is a testspec, +%% Validate that Testspec really is a testspec, %% and exit if not. validate_test(Testspec) -> case lists:member(Testspec, tests()) of @@ -460,10 +638,10 @@ validate_test(Testspec) -> exit(self(), {error, test_not_available}) end. -configvars(Config) -> - case lists:keysearch(vars, 1, Config) of +configvars(Opts) -> + case lists:keysearch(vars, 1, Opts) of {value, {vars, List}} -> - List0 = special_vars(Config), + List0 = special_vars(Opts), Key = fun(T) -> element(1,T) end, DelDupList = lists:filter(fun(V) -> @@ -474,17 +652,17 @@ configvars(Config) -> end, List), {vars, [List0|DelDupList]}; _ -> - {vars, special_vars(Config)} + {vars, special_vars(Opts)} end. -%% Allow some shortcuts in the Options... -special_vars(Config) -> +%% Allow some shortcuts in the options... +special_vars(Opts) -> SpecVars = - case lists:member(verbose, Config) of + case lists:member(verbose, Opts) of true -> [{verbose, 1}]; false -> - case lists:keysearch(verbose, 1, Config) of + case lists:keysearch(verbose, 1, Opts) of {value, {verbose, Lvl}} -> [{verbose, Lvl}]; _ -> @@ -492,13 +670,13 @@ special_vars(Config) -> end end, SpecVars1 = - case lists:keysearch(diskless, 1, Config) of + case lists:keysearch(diskless, 1, Opts) of {value,{diskless, true}} -> [{diskless, true} | SpecVars]; _ -> SpecVars end, - case lists:keysearch(testcase_callback, 1, Config) of + case lists:keysearch(testcase_callback, 1, Opts) of {value,{testcase_callback, CBM, CBF}} -> [{ts_testcase_callback, {CBM,CBF}} | SpecVars1]; {value,{testcase_callback, CB}} -> @@ -566,50 +744,31 @@ check_for_cross_cover_analysis_flag([_|Config],Level,CrossFlag) -> check_for_cross_cover_analysis_flag([],_,_) -> false. -%% Returns a list of available test suites. +%% Returns all available apps. tests() -> {ok, Cwd} = file:get_cwd(), ts_lib:specs(Cwd). -tests(Spec) -> +%% Returns all apps that provide tests in the given test category +tests(main) -> {ok, Cwd} = file:get_cwd(), - ts_lib:suites(Cwd, atom_to_list(Spec)). - -%% Benchmark related functions - -bench() -> - bench([]). - -bench(Opts) when is_list(Opts) -> - bench(benchmarks(),Opts); -bench(Spec) -> - bench([Spec],[]). - -bench(Spec, Opts) when is_atom(Spec) -> - bench([Spec],Opts); -bench(Specs, Opts) -> - check_and_run(fun(Vars) -> ts_benchmark:run(Specs, Opts, Vars) end). - -benchmarks() -> - ts_benchmark:benchmarks(). - -smoke_test() -> - smoke_test([]). - -smoke_test(Opts) when is_list(Opts) -> - smoke_test(smoke_tests(),Opts); -smoke_test(Spec) -> - smoke_test([Spec],[]). - -smoke_test(Spec, Opts) when is_atom(Spec) -> - smoke_test([Spec],Opts); -smoke_test(Specs, Opts) -> - run(Specs, [{smoke,true}|Opts]). + ts_lib:specs(Cwd); +tests(bench) -> + ts_benchmark:benchmarks(); +tests(TestCategory) -> + {ok, Cwd} = file:get_cwd(), + ts_lib:specialized_specs(Cwd, atom_to_list(TestCategory)). + +%% Returns a list of available test suites for App. +suites(App) -> + {ok, Cwd} = file:get_cwd(), + ts_lib:suites(Cwd, atom_to_list(App)). -smoke_tests() -> +%% Returns all available test categories for App +categories(App) -> {ok, Cwd} = file:get_cwd(), - ts_lib:specialized_specs(Cwd,"smoke"). + ts_lib:test_categories(Cwd, atom_to_list(App)). %% %% estone/0, estone/1 diff --git a/lib/test_server/src/ts.hrl b/lib/test_server/src/ts.hrl index 5e829f3575..4c940fdc4f 100644 --- a/lib/test_server/src/ts.hrl +++ b/lib/test_server/src/ts.hrl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2012. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts_autoconf_win32.erl b/lib/test_server/src/ts_autoconf_win32.erl index 4a5c5c7603..288305b406 100644 --- a/lib/test_server/src/ts_autoconf_win32.erl +++ b/lib/test_server/src/ts_autoconf_win32.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2012. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts_benchmark.erl b/lib/test_server/src/ts_benchmark.erl index bd6abc3372..3e55edefb0 100644 --- a/lib/test_server/src/ts_benchmark.erl +++ b/lib/test_server/src/ts_benchmark.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2012-2012. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts_erl_config.erl b/lib/test_server/src/ts_erl_config.erl index 2d45d39700..ab7363c106 100644 --- a/lib/test_server/src/ts_erl_config.erl +++ b/lib/test_server/src/ts_erl_config.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts_install.erl b/lib/test_server/src/ts_install.erl index 594e619fbc..600a576820 100644 --- a/lib/test_server/src/ts_install.erl +++ b/lib/test_server/src/ts_install.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts_install_cth.erl b/lib/test_server/src/ts_install_cth.erl index 7746bbed6f..0462e62611 100644 --- a/lib/test_server/src/ts_install_cth.erl +++ b/lib/test_server/src/ts_install_cth.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 2010-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -40,6 +41,8 @@ -export([post_end_per_group/4]). -export([pre_init_per_testcase/3]). +-export([post_init_per_testcase/4]). +-export([pre_end_per_testcase/3]). -export([post_end_per_testcase/4]). -export([on_tc_fail/3]). @@ -180,7 +183,22 @@ post_end_per_group(_Group,_Config,Return,State) -> pre_init_per_testcase(_TC,Config,State) -> {add_node_name(Config, State), State}. +-spec post_init_per_testcase(TC :: atom(), + Config :: config(), + Return :: term(), + State :: #state{}) -> + {ok | skip_or_fail(), NewState :: #state{}}. +post_init_per_testcase(_TC,_Config,Return,State) -> + {Return, State}. + %% @doc Called after each test case. +-spec pre_end_per_testcase(TC :: atom(), + Config :: config(), + State :: #state{}) -> + {config() | skip_or_fail(), NewState :: #state{}}. +pre_end_per_testcase(_TC,Config,State) -> + {Config, State}. + -spec post_end_per_testcase(TC :: atom(), Config :: config(), Return :: term(), @@ -238,12 +256,15 @@ generate_nodenames2(0, _Hosts, Acc) -> Acc; generate_nodenames2(N, Hosts, Acc) -> Host=lists:nth((N rem (length(Hosts)))+1, Hosts), - Name=list_to_atom(temp_nodename("nod", []) ++ "@" ++ Host), + Name=list_to_atom(temp_nodename("nod",N) ++ "@" ++ Host), generate_nodenames2(N-1, Hosts, [Name|Acc]). -temp_nodename([], Acc) -> - lists:flatten(Acc); -temp_nodename([Chr|Base], Acc) -> - {A,B,C} = erlang:now(), - New = [Chr | integer_to_list(Chr bxor A bxor B+A bxor C+B)], - temp_nodename(Base, [New|Acc]). +%% We cannot use erlang:unique_integer([positive]) +%% here since this code in run on older test releases as well. +temp_nodename(Base,I) -> + {A,B,C} = os:timestamp(), + Nstr = integer_to_list(I), + Astr = integer_to_list(A), + Bstr = integer_to_list(B), + Cstr = integer_to_list(C), + Base++Nstr++Astr++Bstr++Cstr. diff --git a/lib/test_server/src/ts_lib.erl b/lib/test_server/src/ts_lib.erl index 5368960446..7c3f450194 100644 --- a/lib/test_server/src/ts_lib.erl +++ b/lib/test_server/src/ts_lib.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% @@ -27,7 +28,7 @@ erlang_type/1, initial_capital/1, specs/1, suites/2, - specialized_specs/2, + test_categories/2, specialized_specs/2, subst_file/3, subst/2, print_data/1, make_non_erlang/2, maybe_atom_to_list/1, progress/4, @@ -96,26 +97,47 @@ specialized_specs(Dir,PostFix) -> Specs = filelib:wildcard(filename:join([filename:dirname(Dir), "*_test", "*_"++PostFix++".spec"])), sort_tests([begin - Base = filename:basename(Name), - list_to_atom(string:substr(Base,1,string:rstr(Base,"_")-1)) + DirPart = filename:dirname(Name), + AppTest = hd(lists:reverse(filename:split(DirPart))), + list_to_atom(string:substr(AppTest, 1, length(AppTest)-5)) end || Name <- Specs]). specs(Dir) -> Specs = filelib:wildcard(filename:join([filename:dirname(Dir), "*_test", "*.{dyn,}spec"])), - % Filter away all spec which end with {_bench,_smoke}.spec - NoBench = fun(SpecName) -> - case lists:reverse(SpecName) of - "ceps.hcneb_"++_ -> false; - "ceps.ekoms_"++_ -> false; - _ -> true - end - end, - - sort_tests([filename_to_atom(Name) || Name <- Specs, NoBench(Name)]). - -suites(Dir, Spec) -> - Glob=filename:join([filename:dirname(Dir), Spec++"_test", + %% Make sure only to include the main spec for each application + MainSpecs = + lists:flatmap(fun(FullName) -> + [Spec,TestDir|_] = + lists:reverse(filename:split(FullName)), + [_TestSuffix|TDParts] = + lists:reverse(string:tokens(TestDir,[$_,$.])), + [_SpecSuffix|SParts] = + lists:reverse(string:tokens(Spec,[$_,$.])), + if TDParts == SParts -> + [filename_to_atom(FullName)]; + true -> + [] + end + end, Specs), + sort_tests(MainSpecs). + +test_categories(Dir, App) -> + Specs = filelib:wildcard(filename:join([filename:dirname(Dir), + App++"_test", "*.spec"])), + lists:flatmap(fun(FullName) -> + [Spec,_TestDir|_] = + lists:reverse(filename:split(FullName)), + case filename:rootname(Spec -- App) of + "" -> + []; + [_Sep | Cat] -> + [list_to_atom(Cat)] + end + end, Specs). + +suites(Dir, App) -> + Glob=filename:join([filename:dirname(Dir), App++"_test", "*_SUITE.erl"]), Suites=filelib:wildcard(Glob), [filename_to_atom(Name) || Name <- Suites]. @@ -228,12 +250,10 @@ do_test(Rest, Vars, Test) -> {Result,Comment,Rest2}. %% extract an argument -get_arg([$ |Rest], Vars, Stop, Acc) -> - get_arg(Rest, Vars, Stop, Acc); get_arg([$(|Rest], Vars, Stop, _) -> get_arg(Rest, Vars, Stop, []); get_arg([Stop|Rest], Vars, Stop, Acc) -> - Arg = lists:reverse(Acc), + Arg = string:strip(lists:reverse(Acc)), Subst = subst(Arg, Vars), {Subst,Rest}; get_arg([C|Rest], Vars, Stop, Acc) -> diff --git a/lib/test_server/src/ts_make.erl b/lib/test_server/src/ts_make.erl index 9cb77ecb12..0178f4d836 100644 --- a/lib/test_server/src/ts_make.erl +++ b/lib/test_server/src/ts_make.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% diff --git a/lib/test_server/src/ts_run.erl b/lib/test_server/src/ts_run.erl index 18d021f780..188094921d 100644 --- a/lib/test_server/src/ts_run.erl +++ b/lib/test_server/src/ts_run.erl @@ -3,16 +3,17 @@ %% %% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at %% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. %% %% %CopyrightEnd% %% |