diff options
Diffstat (limited to 'lib/common_test/src/ct_run.erl')
| -rw-r--r-- | lib/common_test/src/ct_run.erl | 524 |
1 files changed, 258 insertions, 266 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index acea6b5cb5..c0bdbb2a09 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -330,33 +330,33 @@ script_start1(Parent, Args) -> true end, - StartOpts = #opts{label = Label, profile = Profile, - vts = Vts, shell = Shell, - cover = Cover, cover_stop = CoverStop, - logdir = LogDir, logopts = LogOpts, - basic_html = BasicHtml, - verbosity = Verbosity, - event_handlers = EvHandlers, - ct_hooks = CTHooks, - enable_builtin_hooks = EnableBuiltinHooks, - auto_compile = AutoCompile, - include = IncludeDirs, - silent_connections = SilentConns, - stylesheet = Stylesheet, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir, - starter = script}, - + Opts = #opts{label = Label, profile = Profile, + vts = Vts, shell = Shell, + cover = Cover, cover_stop = CoverStop, + logdir = LogDir, logopts = LogOpts, + basic_html = BasicHtml, + verbosity = Verbosity, + event_handlers = EvHandlers, + ct_hooks = CTHooks, + enable_builtin_hooks = EnableBuiltinHooks, + auto_compile = AutoCompile, + include = IncludeDirs, + silent_connections = SilentConns, + stylesheet = Stylesheet, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir, + starter = script}, + %% check if log files should be refreshed or go on to run tests... - Result = run_or_refresh(StartOpts, Args), + Result = run_or_refresh(Opts, Args), %% send final results to starting process waiting in script_start/0 Parent ! {self(), Result}. -run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> +run_or_refresh(Opts = #opts{logdir = LogDir}, Args) -> case proplists:get_value(refresh_logs, Args) of undefined -> - script_start2(StartOpts, Args); + script_start2(Opts, Args); Refresh -> LogDir1 = case Refresh of [] -> which(logdir,LogDir); @@ -378,171 +378,203 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> {error,{all_suites_index,ASReason}}; _ -> file:set_cwd(Cwd), - io:format("Logs in ~ts refreshed!~n~n", [LogDir1]), + io:format("Logs in ~ts refreshed!~n~n", + [LogDir1]), timer:sleep(500), % time to flush io before quitting ok end end end. -script_start2(StartOpts = #opts{vts = undefined, - shell = undefined}, Args) -> - TestSpec = proplists:get_value(spec, Args), - {Terms,Opts} = - case TestSpec of - Specs when Specs =/= [], Specs =/= undefined -> - %% using testspec as input for test - Relaxed = get_start_opt(allow_user_terms, true, false, Args), - case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of - {E,Reason} when E == error ; E == 'EXIT' -> - {{error,Reason},StartOpts}; - TS -> - SpecStartOpts = get_data_for_node(TS, node()), - - Label = choose_val(StartOpts#opts.label, - SpecStartOpts#opts.label), - - Profile = choose_val(StartOpts#opts.profile, - SpecStartOpts#opts.profile), - - LogDir = choose_val(StartOpts#opts.logdir, - SpecStartOpts#opts.logdir), - - AllLogOpts = merge_vals([StartOpts#opts.logopts, - SpecStartOpts#opts.logopts]), - AllVerbosity = - merge_keyvals([StartOpts#opts.verbosity, - SpecStartOpts#opts.verbosity]), - AllSilentConns = - merge_vals([StartOpts#opts.silent_connections, - SpecStartOpts#opts.silent_connections]), - Cover = - choose_val(StartOpts#opts.cover, - SpecStartOpts#opts.cover), - CoverStop = - choose_val(StartOpts#opts.cover_stop, - SpecStartOpts#opts.cover_stop), - MultTT = - choose_val(StartOpts#opts.multiply_timetraps, - SpecStartOpts#opts.multiply_timetraps), - ScaleTT = - choose_val(StartOpts#opts.scale_timetraps, - SpecStartOpts#opts.scale_timetraps), - - CreatePrivDir = - choose_val(StartOpts#opts.create_priv_dir, - SpecStartOpts#opts.create_priv_dir), - - AllEvHs = - merge_vals([StartOpts#opts.event_handlers, - SpecStartOpts#opts.event_handlers]), - - AllCTHooks = merge_vals( - [StartOpts#opts.ct_hooks, - SpecStartOpts#opts.ct_hooks]), - - EnableBuiltinHooks = - choose_val( - StartOpts#opts.enable_builtin_hooks, - SpecStartOpts#opts.enable_builtin_hooks), - - Stylesheet = - choose_val(StartOpts#opts.stylesheet, - SpecStartOpts#opts.stylesheet), - - AllInclude = merge_vals([StartOpts#opts.include, - SpecStartOpts#opts.include]), - application:set_env(common_test, include, AllInclude), - - AutoCompile = - case choose_val(StartOpts#opts.auto_compile, - SpecStartOpts#opts.auto_compile) of - undefined -> - true; - ACBool -> - application:set_env(common_test, - auto_compile, - ACBool), - ACBool - end, - - BasicHtml = - case choose_val(StartOpts#opts.basic_html, - SpecStartOpts#opts.basic_html) of - undefined -> - false; - BHBool -> - application:set_env(common_test, basic_html, - BHBool), - BHBool - end, - - {TS,StartOpts#opts{label = Label, - profile = Profile, - testspecs = Specs, - cover = Cover, - cover_stop = CoverStop, - logdir = LogDir, - logopts = AllLogOpts, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = SpecStartOpts#opts.config, - event_handlers = AllEvHs, - ct_hooks = AllCTHooks, - enable_builtin_hooks = - EnableBuiltinHooks, - stylesheet = Stylesheet, - auto_compile = AutoCompile, - include = AllInclude, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir}} - end; - _ -> - {undefined,StartOpts} - end, - %% read config/userconfig from start flags - InitConfig = ct_config:prepare_config_list(Args), - TheLogDir = which(logdir, Opts#opts.logdir), - case {TestSpec,Terms} of - {_,{error,_}=Error} -> - Error; - {[],_} -> +script_start2(Opts = #opts{vts = undefined, + shell = undefined}, Args) -> + case proplists:get_value(spec, Args) of + Specs when Specs =/= [], Specs =/= undefined -> + Specs1 = get_start_opt(join_specs, [Specs], Specs, Args), + %% using testspec as input for test + Relaxed = get_start_opt(allow_user_terms, true, false, Args), + case catch ct_testspec:collect_tests_from_file(Specs1, Relaxed) of + {E,Reason} when E == error ; E == 'EXIT' -> + {error,Reason}; + TestSpecData -> + execute_all_specs(TestSpecData, Opts, Args, []) + end; + [] -> {error,no_testspec_specified}; - {undefined,_} -> % no testspec used - case check_and_install_configfiles(InitConfig, TheLogDir, Opts) of + _ -> % no testspec used + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), + case check_and_install_configfiles(InitConfig, + TheLogDir, + Opts) of ok -> % go on read tests from start flags script_start3(Opts#opts{config=InitConfig, logdir=TheLogDir}, Args); Error -> Error - end; - {_,_} -> % testspec used - %% merge config from start flags with config from testspec - AllConfig = merge_vals([InitConfig, Opts#opts.config]), - case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of - ok -> % read tests from spec - {Run,Skip} = ct_testspec:prepare_tests(Terms, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig, - logdir=TheLogDir}, Args); - Error -> - Error end end; -script_start2(StartOpts, Args) -> +script_start2(Opts, Args) -> %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), - LogDir = which(logdir, StartOpts#opts.logdir), - case check_and_install_configfiles(InitConfig, LogDir, StartOpts) of + LogDir = which(logdir, Opts#opts.logdir), + case check_and_install_configfiles(InitConfig, LogDir, Opts) of ok -> % go on read tests from start flags - script_start3(StartOpts#opts{config=InitConfig, - logdir=LogDir}, Args); + script_start3(Opts#opts{config=InitConfig, + logdir=LogDir}, Args); + Error -> + Error + end. + +execute_all_specs([], _, _, Result) -> + Result1 = lists:reverse(Result), + case lists:keysearch('EXIT', 1, Result1) of + {value,{_,_,ExitReason}} -> + exit(ExitReason); + false -> + case lists:keysearch(error, 1, Result1) of + {value,Error} -> + Error; + false -> + lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, + {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> + {Ok1+Ok,Fail1+Fail, + {UserSkip1+UserSkip, + AutoSkip1+AutoSkip}} + end, {0,0,{0,0}}, Result1) + end + end; + +execute_all_specs([{Specs,TS} | TSs], Opts, Args, Result) -> + CombinedOpts = combine_test_opts(TS, Specs, Opts), + try execute_one_spec(TS, CombinedOpts, Args) of + ExecResult -> + execute_all_specs(TSs, Opts, Args, [ExecResult|Result]) + catch + _ : ExitReason -> + execute_all_specs(TSs, Opts, Args, + [{'EXIT',self(),ExitReason}|Result]) + end. + +execute_one_spec(TS, Opts, Args) -> + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), + %% merge config from start flags with config from testspec + AllConfig = merge_vals([InitConfig, Opts#opts.config]), + case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of + ok -> % read tests from spec + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir}, Args); Error -> Error end. +combine_test_opts(TS, Specs, Opts) -> + TSOpts = get_data_for_node(TS, node()), + + Label = choose_val(Opts#opts.label, + TSOpts#opts.label), + + Profile = choose_val(Opts#opts.profile, + TSOpts#opts.profile), + + LogDir = choose_val(Opts#opts.logdir, + TSOpts#opts.logdir), + + AllLogOpts = merge_vals([Opts#opts.logopts, + TSOpts#opts.logopts]), + AllVerbosity = + merge_keyvals([Opts#opts.verbosity, + TSOpts#opts.verbosity]), + AllSilentConns = + merge_vals([Opts#opts.silent_connections, + TSOpts#opts.silent_connections]), + Cover = + choose_val(Opts#opts.cover, + TSOpts#opts.cover), + CoverStop = + choose_val(Opts#opts.cover_stop, + TSOpts#opts.cover_stop), + MultTT = + choose_val(Opts#opts.multiply_timetraps, + TSOpts#opts.multiply_timetraps), + ScaleTT = + choose_val(Opts#opts.scale_timetraps, + TSOpts#opts.scale_timetraps), + + CreatePrivDir = + choose_val(Opts#opts.create_priv_dir, + TSOpts#opts.create_priv_dir), + + AllEvHs = + merge_vals([Opts#opts.event_handlers, + TSOpts#opts.event_handlers]), + + AllCTHooks = merge_vals( + [Opts#opts.ct_hooks, + TSOpts#opts.ct_hooks]), + + EnableBuiltinHooks = + choose_val( + Opts#opts.enable_builtin_hooks, + TSOpts#opts.enable_builtin_hooks), + + Stylesheet = + choose_val(Opts#opts.stylesheet, + TSOpts#opts.stylesheet), + + AllInclude = merge_vals([Opts#opts.include, + TSOpts#opts.include]), + application:set_env(common_test, include, AllInclude), + + AutoCompile = + case choose_val(Opts#opts.auto_compile, + TSOpts#opts.auto_compile) of + undefined -> + true; + ACBool -> + application:set_env(common_test, + auto_compile, + ACBool), + ACBool + end, + + BasicHtml = + case choose_val(Opts#opts.basic_html, + TSOpts#opts.basic_html) of + undefined -> + false; + BHBool -> + application:set_env(common_test, basic_html, + BHBool), + BHBool + end, + + Opts#opts{label = Label, + profile = Profile, + testspecs = Specs, + cover = Cover, + cover_stop = CoverStop, + logdir = which(logdir, LogDir), + logopts = AllLogOpts, + basic_html = BasicHtml, + verbosity = AllVerbosity, + silent_connections = AllSilentConns, + config = TSOpts#opts.config, + event_handlers = AllEvHs, + ct_hooks = AllCTHooks, + enable_builtin_hooks = EnableBuiltinHooks, + stylesheet = Stylesheet, + auto_compile = AutoCompile, + include = AllInclude, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir}. + check_and_install_configfiles( Configs, LogDir, #opts{ event_handlers = EvHandlers, @@ -562,12 +594,12 @@ check_and_install_configfiles( {error,{cant_load_callback_module,Info}} end. -script_start3(StartOpts, Args) -> - StartOpts1 = get_start_opt(step, - fun(Step) -> - StartOpts#opts{step = Step, - cover = undefined} - end, StartOpts, Args), +script_start3(Opts, Args) -> + Opts1 = get_start_opt(step, + fun(Step) -> + Opts#opts{step = Step, + cover = undefined} + end, Opts, Args), case {proplists:get_value(dir, Args), proplists:get_value(suite, Args), groups_and_cases(proplists:get_value(group, Args), @@ -581,17 +613,17 @@ script_start3(StartOpts, Args) -> {error,no_dir_specified}; {Dirs,undefined,[]} when is_list(Dirs) -> - script_start4(StartOpts#opts{tests = tests(Dirs)}, Args); + script_start4(Opts#opts{tests = tests(Dirs)}, Args); {undefined,Suites,[]} when is_list(Suites) -> Ts = tests([suite_to_test(S) || S <- Suites]), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); {undefined,Suite,GsAndCs} when is_list(Suite) -> case [suite_to_test(S) || S <- Suite] of DirMods = [_] -> Ts = tests(DirMods, GsAndCs), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); [_,_|_] -> {error,multiple_suites_and_cases}; _ -> @@ -605,10 +637,10 @@ script_start3(StartOpts, Args) -> case [suite_to_test(Dir,S) || S <- Suite] of DirMods when GsAndCs == [] -> Ts = tests(DirMods), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); DirMods = [_] when GsAndCs /= [] -> Ts = tests(DirMods, GsAndCs), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); [_,_|_] when GsAndCs /= [] -> {error,multiple_suites_and_cases}; _ -> @@ -619,8 +651,8 @@ script_start3(StartOpts, Args) -> {error,incorrect_start_options}; {undefined,undefined,_} -> - if StartOpts#opts.vts ; StartOpts#opts.shell -> - script_start4(StartOpts#opts{tests = []}, Args); + if Opts#opts.vts ; Opts#opts.shell -> + script_start4(Opts#opts{tests = []}, Args); true -> script_usage(), {error,missing_start_options} @@ -750,6 +782,7 @@ script_usage() -> "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" "\n\t[-allow_user_terms]" + "\n\t[-join_specs]" "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" @@ -1044,105 +1077,60 @@ run_test2(StartOpts) -> end. run_spec_file(Relaxed, - Opts = #opts{testspecs = Specs, config = CfgFiles}, + Opts = #opts{testspecs = Specs}, StartOpts) -> Specs1 = case Specs of [X|_] when is_integer(X) -> [Specs]; _ -> Specs end, AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), - log_ts_names(AbsSpecs), - case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of + AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts), + case catch ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of {Error,CTReason} when Error == error ; Error == 'EXIT' -> exit({error,CTReason}); - TS -> - SpecOpts = get_data_for_node(TS, node()), - Label = choose_val(Opts#opts.label, - SpecOpts#opts.label), - Profile = choose_val(Opts#opts.profile, - SpecOpts#opts.profile), - LogDir = choose_val(Opts#opts.logdir, - SpecOpts#opts.logdir), - AllLogOpts = merge_vals([Opts#opts.logopts, - SpecOpts#opts.logopts]), - Stylesheet = choose_val(Opts#opts.stylesheet, - SpecOpts#opts.stylesheet), - AllVerbosity = merge_keyvals([Opts#opts.verbosity, - SpecOpts#opts.verbosity]), - AllSilentConns = merge_vals([Opts#opts.silent_connections, - SpecOpts#opts.silent_connections]), - AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), - Cover = choose_val(Opts#opts.cover, - SpecOpts#opts.cover), - CoverStop = choose_val(Opts#opts.cover_stop, - SpecOpts#opts.cover_stop), - MultTT = choose_val(Opts#opts.multiply_timetraps, - SpecOpts#opts.multiply_timetraps), - ScaleTT = choose_val(Opts#opts.scale_timetraps, - SpecOpts#opts.scale_timetraps), - CreatePrivDir = choose_val(Opts#opts.create_priv_dir, - SpecOpts#opts.create_priv_dir), - AllEvHs = merge_vals([Opts#opts.event_handlers, - SpecOpts#opts.event_handlers]), - AllInclude = merge_vals([Opts#opts.include, - SpecOpts#opts.include]), - AllCTHooks = merge_vals([Opts#opts.ct_hooks, - SpecOpts#opts.ct_hooks]), - EnableBuiltinHooks = choose_val(Opts#opts.enable_builtin_hooks, - SpecOpts#opts.enable_builtin_hooks), - - application:set_env(common_test, include, AllInclude), - - AutoCompile = case choose_val(Opts#opts.auto_compile, - SpecOpts#opts.auto_compile) of - undefined -> - true; - ACBool -> - application:set_env(common_test, auto_compile, - ACBool), - ACBool - end, + TestSpecData -> + run_all_specs(TestSpecData, Opts, StartOpts, []) + end. - BasicHtml = case choose_val(Opts#opts.basic_html, - SpecOpts#opts.basic_html) of - undefined -> - false; - BHBool -> - application:set_env(common_test, basic_html, - BHBool), - BHBool - end, - - Opts1 = Opts#opts{label = Label, - profile = Profile, - cover = Cover, - cover_stop = CoverStop, - logdir = which(logdir, LogDir), - logopts = AllLogOpts, - stylesheet = Stylesheet, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = AllConfig, - event_handlers = AllEvHs, - auto_compile = AutoCompile, - include = AllInclude, - testspecs = AbsSpecs, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir, - ct_hooks = AllCTHooks, - enable_builtin_hooks = EnableBuiltinHooks - }, - - case check_and_install_configfiles(AllConfig,Opts1#opts.logdir, - Opts1) of - ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); - {error,GCFReason} -> - exit({error,GCFReason}) +run_all_specs([], _, _, TotResult) -> + TotResult1 = lists:reverse(TotResult), + case lists:keysearch('EXIT', 1, TotResult1) of + {value,{_,_,ExitReason}} -> + exit(ExitReason); + false -> + case lists:keysearch(error, 1, TotResult1) of + {value,Error} -> + Error; + false -> + lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, + {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> + {Ok1+Ok,Fail1+Fail, + {UserSkip1+UserSkip, + AutoSkip1+AutoSkip}} + end, {0,0,{0,0}}, TotResult1) end + end; + +run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) -> + log_ts_names(Specs), + Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts), + AllConfig = merge_vals([Opts#opts.config, TSConfig]), + try run_one_spec(TS, Combined#opts{config = AllConfig}, StartOpts) of + Result -> + run_all_specs(TSs, Opts, StartOpts, [Result | TotResult]) + catch + _ : Reason -> + run_all_specs(TSs, Opts, StartOpts, [{error,Reason} | TotResult]) + end. + +run_one_spec(TS, CombinedOpts, StartOpts) -> + #opts{logdir = Logdir, config = Config} = CombinedOpts, + case check_and_install_configfiles(Config, Logdir, CombinedOpts) of + ok -> + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + reformat_result(catch do_run(Run, Skip, CombinedOpts, StartOpts)); + Error -> + Error end. run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, @@ -2896,6 +2884,10 @@ opts2args(EnvStartOpts) -> [{allow_user_terms,[]}]; ({allow_user_terms,false}) -> []; + ({join_specs,true}) -> + [{join_specs,[]}]; + ({join_specs,false}) -> + []; ({auto_compile,false}) -> [{no_auto_compile,[]}]; ({auto_compile,true}) -> |