diff options
Diffstat (limited to 'lib/common_test/src/ct_run.erl')
| -rw-r--r-- | lib/common_test/src/ct_run.erl | 205 |
1 files changed, 143 insertions, 62 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 3383244bf4..eb05c90ba8 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -58,6 +58,7 @@ vts, shell, cover, + cover_stop, coverspec, step, logdir, @@ -245,6 +246,7 @@ script_start1(Parent, Args) -> Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), + CoverStop = get_start_opt(cover_stop, fun([CS]) -> list_to_atom(CS) end, Args), LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), LogOpts = get_start_opt(logopts, fun(Os) -> [list_to_atom(O) || O <- Os] end, [], Args), @@ -329,7 +331,8 @@ script_start1(Parent, Args) -> end, StartOpts = #opts{label = Label, profile = Profile, - vts = Vts, shell = Shell, cover = Cover, + vts = Vts, shell = Shell, + cover = Cover, cover_stop = CoverStop, logdir = LogDir, logopts = LogOpts, basic_html = BasicHtml, verbosity = Verbosity, @@ -416,6 +419,9 @@ script_start2(StartOpts = #opts{vts = undefined, Cover = choose_val(StartOpts#opts.cover, SpecStartOpts#opts.cover), + CoverStop = + choose_val(StartOpts#opts.cover_stop, + SpecStartOpts#opts.cover_stop), MultTT = choose_val(StartOpts#opts.multiply_timetraps, SpecStartOpts#opts.multiply_timetraps), @@ -475,6 +481,7 @@ script_start2(StartOpts = #opts{vts = undefined, profile = Profile, testspecs = Specs, cover = Cover, + cover_stop = CoverStop, logdir = LogDir, logopts = AllLogOpts, basic_html = BasicHtml, @@ -723,6 +730,7 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" + "\n\t[-cover_stop Bool]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" @@ -745,6 +753,7 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" + "\n\t[-cover_stop Bool]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" @@ -938,6 +947,7 @@ run_test2(StartOpts) -> %% code coverage Cover = get_start_opt(cover, fun(CoverFile) -> ?abs(CoverFile) end, StartOpts), + CoverStop = get_start_opt(cover_stop, value, StartOpts), %% timetrap manipulation MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts), @@ -1000,7 +1010,8 @@ run_test2(StartOpts) -> Step = get_start_opt(step, value, StartOpts), Opts = #opts{label = Label, profile = Profile, - cover = Cover, step = Step, logdir = LogDir, + cover = Cover, cover_stop = CoverStop, + step = Step, logdir = LogDir, logopts = LogOpts, basic_html = BasicHtml, config = CfgFiles, verbosity = Verbosity, @@ -1063,6 +1074,8 @@ run_spec_file(Relaxed, AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), Cover = choose_val(Opts#opts.cover, SpecOpts#opts.cover), + CoverStop = choose_val(Opts#opts.cover_stop, + SpecOpts#opts.cover_stop), MultTT = choose_val(Opts#opts.multiply_timetraps, SpecOpts#opts.multiply_timetraps), ScaleTT = choose_val(Opts#opts.scale_timetraps, @@ -1103,6 +1116,7 @@ run_spec_file(Relaxed, Opts1 = Opts#opts{label = Label, profile = Profile, cover = Cover, + cover_stop = CoverStop, logdir = which(logdir, LogDir), logopts = AllLogOpts, stylesheet = Stylesheet, @@ -1272,7 +1286,8 @@ run_dir(Opts = #opts{logdir = LogDir, reformat_result(catch do_run(tests(Dir2, Mod), [], Opts1, StartOpts)); _ -> - reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), + reformat_result(catch do_run(tests(Dir2, Mod, + GsAndCs), [], Opts1, StartOpts)) end; @@ -1281,7 +1296,8 @@ run_dir(Opts = #opts{logdir = LogDir, [_,_|_] when GsAndCs /= [] -> exit({error,multiple_suites_and_cases}); [{Dir2,Mod}] when GsAndCs /= [] -> - reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), + reformat_result(catch do_run(tests(Dir2, Mod, + GsAndCs), [], Opts1, StartOpts)); DirMods -> reformat_result(catch do_run(tests(DirMods), @@ -1374,6 +1390,7 @@ get_data_for_node(#testspec{label = Labels, verbosity = VLvls, silent_connections = SilentConnsList, cover = CoverFs, + cover_stop = CoverStops, config = Cfgs, userconfig = UsrCfgs, event_handler = EvHs, @@ -1405,6 +1422,7 @@ get_data_for_node(#testspec{label = Labels, SCs -> SCs end, Cover = proplists:get_value(Node, CoverFs), + CoverStop = proplists:get_value(Node, CoverStops), MT = proplists:get_value(Node, MTs), ST = proplists:get_value(Node, STs), CreatePrivDir = proplists:get_value(Node, PDs), @@ -1423,6 +1441,7 @@ get_data_for_node(#testspec{label = Labels, verbosity = Verbosity, silent_connections = SilentConns, cover = Cover, + cover_stop = CoverStop, config = ConfigFiles, event_handlers = EvHandlers, ct_hooks = FiltCTHooks, @@ -1536,17 +1555,36 @@ groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and ((Cs == undefined) or (Cs == [])) -> []; groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] -> - [ensure_atom(C) || C <- listify(Cs)]; -groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] -> - [{ensure_atom(G),all} || G <- listify(Gs)]; -groups_and_cases(G, Cs) when is_atom(G) -> - [{G,[ensure_atom(C) || C <- listify(Cs)]}]; -groups_and_cases([G], Cs) -> - [{ensure_atom(G),[ensure_atom(C) || C <- listify(Cs)]}]; -groups_and_cases([_,_|_] , Cs) when Cs =/= [] -> - {error,multiple_groups_and_cases}; -groups_and_cases(_Gs, _Cs) -> - {error,incorrect_group_or_case_option}. + if (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> all; + true -> [ensure_atom(C) || C <- listify(Cs)] + end; +groups_and_cases(GOrGs, Cs) when (is_atom(GOrGs) orelse + (is_list(GOrGs) andalso + (is_atom(hd(GOrGs)) orelse + (is_list(hd(GOrGs)) andalso + is_atom(hd(hd(GOrGs))))))) -> + if (Cs == undefined) or (Cs == []) or + (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> + [{GOrGs,all}]; + true -> + [{GOrGs,[ensure_atom(C) || C <- listify(Cs)]}] + end; +groups_and_cases(Gs, Cs) when is_integer(hd(hd(Gs))) -> + %% if list of strings, this comes from 'ct_run -group G1 G2 ...' and + %% we need to parse the strings + Gs1 = + if (Gs == [all]) or (Gs == ["all"]) -> + all; + true -> + lists:map(fun(G) -> + {ok,Ts,_} = erl_scan:string(G++"."), + {ok,Term} = erl_parse:parse_term(Ts), + Term + end, Gs) + end, + groups_and_cases(Gs1, Cs); +groups_and_cases(Gs, Cs) -> + {error,{incorrect_group_or_case_option,Gs,Cs}}. tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) -> [{?testdir(TestDir,Suites),ensure_atom(Suites),all}]; @@ -1576,14 +1614,7 @@ do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), StepOpts -> #opts{step = StepOpts} end, - Opts1 = - case proplists:get_value(cover, Misc) of - undefined -> - Opts; - CoverFile -> - Opts#opts{cover = CoverFile} - end, - do_run(Tests, [], Opts1#opts{logdir = LogDir}, []); + do_run(Tests, [], Opts#opts{logdir = LogDir}, []); do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> #opts{label = Label, profile = Profile, cover = Cover, @@ -1617,7 +1648,13 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> {error,Reason} -> exit({error,Reason}); CoverSpec -> - Opts#opts{coverspec = CoverSpec} + CoverStop = + case Opts#opts.cover_stop of + undefined -> true; + Stop -> Stop + end, + Opts#opts{coverspec = CoverSpec, + cover_stop = CoverStop} end end, %% This env variable is used by test_server to determine @@ -1687,11 +1724,15 @@ compile_and_run(Tests, Skip, Opts, Args) -> SavedErrors = save_make_errors(SuiteMakeErrors), ct_repeat:log_loop_info(Args), - {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors), - - ReleaseSh = proplists:get_value(release_shell, Args), - ct_util:set_testdata({release_shell,ReleaseSh}), - possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts); + try final_tests(Tests,Skip,SavedErrors) of + {Tests1,Skip1} -> + ReleaseSh = proplists:get_value(release_shell, Args), + ct_util:set_testdata({release_shell,ReleaseSh}), + possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts) + catch + _:BadFormat -> + {error,BadFormat} + end; false -> io:nl(), ct_util:stop(clean), @@ -1961,22 +2002,21 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when %% for now, only flat group defs are allowed as %% start options and test spec terms fun({all,all}) -> - ct_framework:make_all_conf(TestDir, - Suite, []); + [ct_groups:make_conf(TestDir, Suite, all, [], all)]; ({skipped,Group,TCs}) -> - [ct_framework:make_conf(TestDir, Suite, - Group, [skipped], TCs)]; - ({GrSpec = {Group,_},TCs}) -> + [ct_groups:make_conf(TestDir, Suite, + Group, [skipped], TCs)]; + ({GrSpec = {GroupName,_},TCs}) -> Props = [{override,GrSpec}], - [ct_framework:make_conf(TestDir, Suite, - Group, Props, TCs)]; - ({GrSpec = {Group,_,_},TCs}) -> + [ct_groups:make_conf(TestDir, Suite, + GroupName, Props, TCs)]; + ({GrSpec = {GroupName,_,_},TCs}) -> Props = [{override,GrSpec}], - [ct_framework:make_conf(TestDir, Suite, - Group, Props, TCs)]; - ({Group,TCs}) -> - [ct_framework:make_conf(TestDir, Suite, - Group, [], TCs)]; + [ct_groups:make_conf(TestDir, Suite, + GroupName, Props, TCs)]; + ({GroupOrGroups,TCs}) -> + [ct_groups:make_conf(TestDir, Suite, + GroupOrGroups, [], TCs)]; (TC) -> [TC] end, GrsOrCs), @@ -1988,12 +2028,12 @@ final_tests1([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) -> - SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all), + SkipConf = ct_groups:make_conf(TestDir, Suite, all, [], all), Skip = {TestDir,Suite,SkipConf,Reason}, final_skip(Skips, [Skip|Final]); final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> - Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs), + Conf = ct_groups:make_conf(TestDir, Suite, Group, [], TCs), Skip = {TestDir,Suite,Conf,Reason}, final_skip(Skips, [Skip|Final]); @@ -2120,7 +2160,8 @@ do_run_test(Tests, Skip, Opts) -> %% tell test_server which modules should be cover compiled %% note that actual compilation is done when tests start test_server_ctrl:cover(CovApp, CovFile, CovExcl, CovIncl, - CovCross, CovExport, CovLevel), + CovCross, CovExport, CovLevel, + Opts#opts.cover_stop), %% save cover data (used e.g. to add nodes dynamically) ct_util:set_testdata({cover,CovData}), %% start cover on specified nodes @@ -2192,6 +2233,15 @@ do_run_test(Tests, Skip, Opts) -> end, CleanUp), [code:del_path(Dir) || Dir <- AddedToPath], + %% If a severe error has occurred in the test_server, + %% we will generate an exception here. + case ct_util:get_testdata(severe_error) of + undefined -> ok; + SevereError -> + ct_logs:log("SEVERE ERROR", "~p\n", [SevereError]), + exit(SevereError) + end, + case ct_util:get_testdata(stats) of Stats = {_Ok,_Failed,{_UserSkipped,_AutoSkipped}} -> Stats; @@ -2256,9 +2306,11 @@ add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) -> wait_for_idle(), add_jobs(Tests, Skip, Opts, CleanUp) end; -add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) -> +add_jobs([{TestDir,[Suite],all}|Tests], Skip, + Opts, CleanUp) when is_atom(Suite) -> add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp); -add_jobs([{TestDir,Suites,all}|Tests], Skip, Opts, CleanUp) when is_list(Suites) -> +add_jobs([{TestDir,Suites,all}|Tests], Skip, + Opts, CleanUp) when is_list(Suites) -> Name = get_name(TestDir) ++ ".suites", case catch test_server_ctrl:add_module_with_skip(Name, Suites, skiplist(TestDir,Skip)) of @@ -2273,7 +2325,8 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> ok -> Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite), case catch test_server_ctrl:add_module_with_skip(Name, [Suite], - skiplist(TestDir,Skip)) of + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2296,15 +2349,24 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when GrTestName = case Confs of [Conf] -> - "." ++ atom_to_list(Group(Conf)) ++ TCTestName(TestCases(Conf)); + case Group(Conf) of + GrName when is_atom(GrName) -> + "." ++ atom_to_list(GrName) ++ + TCTestName(TestCases(Conf)); + _ -> + ".groups" ++ TCTestName(TestCases(Conf)) + end; _ -> ".groups" end, TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName, case maybe_interpret(Suite, init_per_group, Opts) of ok -> - case catch test_server_ctrl:add_conf_with_skip(TestName, Suite, Confs, - skiplist(TestDir,Skip)) of + case catch test_server_ctrl:add_conf_with_skip(TestName, + Suite, + Confs, + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2316,18 +2378,21 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when end; %% test case -add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -> +add_jobs([{TestDir,Suite,[Case]}|Tests], + Skip, Opts, CleanUp) when is_atom(Case) -> add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp); -add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opts, CleanUp) when is_list(Cases) -> +add_jobs([{TestDir,Suite,Cases}|Tests], + Skip, Opts, CleanUp) when is_list(Cases) -> Cases1 = lists:map(fun({GroupName,_}) when is_atom(GroupName) -> GroupName; (Case) -> Case end, Cases), case maybe_interpret(Suite, Cases1, Opts) of ok -> - Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases", + Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases", case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases1, - skiplist(TestDir,Skip)) of + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2343,7 +2408,8 @@ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) - Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++ atom_to_list(Case), case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case, - skiplist(TestDir,Skip)) of + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2378,7 +2444,8 @@ skiplist(Dir, [{Dir,all,Cmt}|Skip]) -> %% we need to turn 'all' into list of modules since %% test_server doesn't do skips on Dir level Ss = filelib:wildcard(filename:join(Dir, "*_SUITE.beam")), - [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ skiplist(Dir,Skip); + [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ + skiplist(Dir,Skip); skiplist(Dir, [{Dir,S,Cmt}|Skip]) -> [{S,Cmt} | skiplist(Dir, Skip)]; skiplist(Dir, [{Dir,S,C,Cmt}|Skip]) -> @@ -2438,8 +2505,10 @@ run_make(Targets, TestDir0, Mod, UserInclude) -> FileTest = fun(F, suites) -> is_suite(F); (F, helpmods) -> not is_suite(F) end, - Files = lists:flatmap(fun({F,out_of_date}) -> - case FileTest(F, Targets) of + Files = + lists:flatmap(fun({F,out_of_date}) -> + case FileTest(F, + Targets) of true -> [F]; false -> [] end; @@ -2575,6 +2644,9 @@ merge_arguments([LogDir={logdir,_}|Args], Merged) -> merge_arguments([CoverFile={cover,_}|Args], Merged) -> merge_arguments(Args, handle_arg(replace, CoverFile, Merged)); +merge_arguments([CoverStop={cover_stop,_}|Args], Merged) -> + merge_arguments(Args, handle_arg(replace, CoverStop, Merged)); + merge_arguments([{'case',TC}|Args], Merged) -> merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged)); @@ -2783,11 +2855,14 @@ opts2args(EnvStartOpts) -> lists:flatmap(fun({exit_status,ExitStatusOpt}) when is_atom(ExitStatusOpt) -> [{exit_status,[atom_to_list(ExitStatusOpt)]}]; ({halt_with,{HaltM,HaltF}}) -> - [{halt_with,[atom_to_list(HaltM),atom_to_list(HaltF)]}]; + [{halt_with,[atom_to_list(HaltM), + atom_to_list(HaltF)]}]; ({interactive_mode,true}) -> [{shell,[]}]; - ({config,CfgFiles}) -> - [{ct_config,[CfgFiles]}]; + ({config,CfgFile}) when is_integer(hd(CfgFile)) -> + [{ct_config,[CfgFile]}]; + ({config,CfgFiles}) when is_list(hd(CfgFiles)) -> + [{ct_config,CfgFiles}]; ({userconfig,{CBM,CfgStr=[X|_]}}) when is_integer(X) -> [{userconfig,[atom_to_list(CBM),CfgStr]}]; ({userconfig,{CBM,CfgStrs}}) when is_list(CfgStrs) -> @@ -2805,6 +2880,12 @@ opts2args(EnvStartOpts) -> end, UserCfg), [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), [{userconfig,lists:reverse(StrsR)}]; + ({group,G}) when is_atom(G) -> + [{group,[atom_to_list(G)]}]; + ({group,Gs}) when is_list(Gs) -> + LOfGStrs = [lists:flatten(io_lib:format("~w",[G])) || + G <- Gs], + [{group,LOfGStrs}]; ({testcase,Case}) when is_atom(Case) -> [{'case',[atom_to_list(Case)]}]; ({testcase,Cases}) -> |