From adc50b681c3d3bd6d98f6c3f88dad4a306d3a239 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 11 Jan 2013 13:13:32 +0100 Subject: Make it possible to execute one test run per test specification OTP-9881 --- lib/common_test/src/ct_run.erl | 307 ++++++++++++++++++++---------------- lib/common_test/src/ct_testspec.erl | 32 +++- 2 files changed, 194 insertions(+), 145 deletions(-) (limited to 'lib/common_test/src') diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index eb05c90ba8..50723a763c 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -378,7 +378,8 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> {error,{all_suites_index,ASReason}}; _ -> file:set_cwd(Cwd), - io:format("Logs in ~s refreshed!~n~n", [LogDir1]), + io:format("Logs in ~s refreshed!~n~n", + [LogDir1]), timer:sleep(500), % time to flush io before quitting ok end @@ -387,145 +388,29 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> script_start2(StartOpts = #opts{vts = undefined, shell = undefined}, Args) -> - TestSpec = proplists:get_value(spec, Args), - {Terms,Opts} = - case TestSpec of - Specs when Specs =/= [], Specs =/= undefined -> - %% using testspec as input for test - Relaxed = get_start_opt(allow_user_terms, true, false, Args), - case catch ct_testspec:collect_tests_from_file(Specs, Relaxed) of - {E,Reason} when E == error ; E == 'EXIT' -> - {{error,Reason},StartOpts}; - TS -> - SpecStartOpts = get_data_for_node(TS, node()), - - Label = choose_val(StartOpts#opts.label, - SpecStartOpts#opts.label), - - Profile = choose_val(StartOpts#opts.profile, - SpecStartOpts#opts.profile), - - LogDir = choose_val(StartOpts#opts.logdir, - SpecStartOpts#opts.logdir), - - AllLogOpts = merge_vals([StartOpts#opts.logopts, - SpecStartOpts#opts.logopts]), - AllVerbosity = - merge_keyvals([StartOpts#opts.verbosity, - SpecStartOpts#opts.verbosity]), - AllSilentConns = - merge_vals([StartOpts#opts.silent_connections, - SpecStartOpts#opts.silent_connections]), - Cover = - choose_val(StartOpts#opts.cover, - SpecStartOpts#opts.cover), - CoverStop = - choose_val(StartOpts#opts.cover_stop, - SpecStartOpts#opts.cover_stop), - MultTT = - choose_val(StartOpts#opts.multiply_timetraps, - SpecStartOpts#opts.multiply_timetraps), - ScaleTT = - choose_val(StartOpts#opts.scale_timetraps, - SpecStartOpts#opts.scale_timetraps), - - CreatePrivDir = - choose_val(StartOpts#opts.create_priv_dir, - SpecStartOpts#opts.create_priv_dir), - - AllEvHs = - merge_vals([StartOpts#opts.event_handlers, - SpecStartOpts#opts.event_handlers]), - - AllCTHooks = merge_vals( - [StartOpts#opts.ct_hooks, - SpecStartOpts#opts.ct_hooks]), - - EnableBuiltinHooks = - choose_val( - StartOpts#opts.enable_builtin_hooks, - SpecStartOpts#opts.enable_builtin_hooks), - - Stylesheet = - choose_val(StartOpts#opts.stylesheet, - SpecStartOpts#opts.stylesheet), - - AllInclude = merge_vals([StartOpts#opts.include, - SpecStartOpts#opts.include]), - application:set_env(common_test, include, AllInclude), - - AutoCompile = - case choose_val(StartOpts#opts.auto_compile, - SpecStartOpts#opts.auto_compile) of - undefined -> - true; - ACBool -> - application:set_env(common_test, - auto_compile, - ACBool), - ACBool - end, - - BasicHtml = - case choose_val(StartOpts#opts.basic_html, - SpecStartOpts#opts.basic_html) of - undefined -> - false; - BHBool -> - application:set_env(common_test, basic_html, - BHBool), - BHBool - end, - - {TS,StartOpts#opts{label = Label, - profile = Profile, - testspecs = Specs, - cover = Cover, - cover_stop = CoverStop, - logdir = LogDir, - logopts = AllLogOpts, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = SpecStartOpts#opts.config, - event_handlers = AllEvHs, - ct_hooks = AllCTHooks, - enable_builtin_hooks = - EnableBuiltinHooks, - stylesheet = Stylesheet, - auto_compile = AutoCompile, - include = AllInclude, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir}} - end; - _ -> - {undefined,StartOpts} - end, - %% read config/userconfig from start flags - InitConfig = ct_config:prepare_config_list(Args), - TheLogDir = which(logdir, Opts#opts.logdir), - case {TestSpec,Terms} of - {_,{error,_}=Error} -> - Error; - {[],_} -> + case proplists:get_value(spec, Args) of + Specs when Specs =/= [], Specs =/= undefined -> + Specs1 = get_start_opt(join_specs, [Specs], Specs, Args), + %% using testspec as input for test + Relaxed = get_start_opt(allow_user_terms, true, false, Args), + case catch ct_testspec:collect_tests_from_file(Specs1, Relaxed) of + {E,Reason} when E == error ; E == 'EXIT' -> + {error,Reason}; + TestSpecData -> + execute_testspecs(TestSpecData, StartOpts, Args, []) + end; + [] -> {error,no_testspec_specified}; - {undefined,_} -> % no testspec used - case check_and_install_configfiles(InitConfig, TheLogDir, Opts) of + _ -> % no testspec used + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, StartOpts#opts.logdir), + case check_and_install_configfiles(InitConfig, + TheLogDir, + StartOpts) of ok -> % go on read tests from start flags - script_start3(Opts#opts{config=InitConfig, - logdir=TheLogDir}, Args); - Error -> - Error - end; - {_,_} -> % testspec used - %% merge config from start flags with config from testspec - AllConfig = merge_vals([InitConfig, Opts#opts.config]), - case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of - ok -> % read tests from spec - {Run,Skip} = ct_testspec:prepare_tests(Terms, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig, - logdir=TheLogDir}, Args); + script_start3(StartOpts#opts{config=InitConfig, + logdir=TheLogDir}, Args); Error -> Error end @@ -543,6 +428,152 @@ script_start2(StartOpts, Args) -> Error end. +execute_testspecs([], _, _, Result) -> + Result1 = lists:reverse(Result), + case lists:keysearch('EXIT', 1, Result1) of + {value,{_,_,ExitReason}} -> + exit(ExitReason); + false -> + case lists:keysearch(error, 1, Result1) of + {value,Error} -> + Error; + false -> + lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, + {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> + {Ok1+Ok,Fail1+Fail, + {UserSkip1+UserSkip, + AutoSkip1+AutoSkip}} + end, {0,0,{0,0}}, Result1) + end + end; + +execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> + SpecStartOpts = get_data_for_node(TS, node()), + + Label = choose_val(StartOpts#opts.label, + SpecStartOpts#opts.label), + + Profile = choose_val(StartOpts#opts.profile, + SpecStartOpts#opts.profile), + + LogDir = choose_val(StartOpts#opts.logdir, + SpecStartOpts#opts.logdir), + + AllLogOpts = merge_vals([StartOpts#opts.logopts, + SpecStartOpts#opts.logopts]), + AllVerbosity = + merge_keyvals([StartOpts#opts.verbosity, + SpecStartOpts#opts.verbosity]), + AllSilentConns = + merge_vals([StartOpts#opts.silent_connections, + SpecStartOpts#opts.silent_connections]), + Cover = + choose_val(StartOpts#opts.cover, + SpecStartOpts#opts.cover), + CoverStop = + choose_val(StartOpts#opts.cover_stop, + SpecStartOpts#opts.cover_stop), + MultTT = + choose_val(StartOpts#opts.multiply_timetraps, + SpecStartOpts#opts.multiply_timetraps), + ScaleTT = + choose_val(StartOpts#opts.scale_timetraps, + SpecStartOpts#opts.scale_timetraps), + + CreatePrivDir = + choose_val(StartOpts#opts.create_priv_dir, + SpecStartOpts#opts.create_priv_dir), + + AllEvHs = + merge_vals([StartOpts#opts.event_handlers, + SpecStartOpts#opts.event_handlers]), + + AllCTHooks = merge_vals( + [StartOpts#opts.ct_hooks, + SpecStartOpts#opts.ct_hooks]), + + EnableBuiltinHooks = + choose_val( + StartOpts#opts.enable_builtin_hooks, + SpecStartOpts#opts.enable_builtin_hooks), + + Stylesheet = + choose_val(StartOpts#opts.stylesheet, + SpecStartOpts#opts.stylesheet), + + AllInclude = merge_vals([StartOpts#opts.include, + SpecStartOpts#opts.include]), + application:set_env(common_test, include, AllInclude), + + AutoCompile = + case choose_val(StartOpts#opts.auto_compile, + SpecStartOpts#opts.auto_compile) of + undefined -> + true; + ACBool -> + application:set_env(common_test, + auto_compile, + ACBool), + ACBool + end, + + BasicHtml = + case choose_val(StartOpts#opts.basic_html, + SpecStartOpts#opts.basic_html) of + undefined -> + false; + BHBool -> + application:set_env(common_test, basic_html, + BHBool), + BHBool + end, + + Opts = StartOpts#opts{label = Label, + profile = Profile, + testspecs = Specs, + cover = Cover, + cover_stop = CoverStop, + logdir = LogDir, + logopts = AllLogOpts, + basic_html = BasicHtml, + verbosity = AllVerbosity, + silent_connections = AllSilentConns, + config = SpecStartOpts#opts.config, + event_handlers = AllEvHs, + ct_hooks = AllCTHooks, + enable_builtin_hooks = + EnableBuiltinHooks, + stylesheet = Stylesheet, + auto_compile = AutoCompile, + include = AllInclude, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir}, + + try execute_testspec(TS, Opts, Args) of + ExecResult -> + execute_testspecs(TSs, StartOpts, Args, [ExecResult | Result]) + catch + _ : ExitReason -> + execute_testspecs(TSs, StartOpts, Args, + [{'EXIT',self(),ExitReason} | Result]) + end. + +execute_testspec(TS, Opts, Args) -> + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), + %% merge config from start flags with config from testspec + AllConfig = merge_vals([InitConfig, Opts#opts.config]), + case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of + ok -> % read tests from spec + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir}, Args); + Error -> + Error + end. + check_and_install_configfiles( Configs, LogDir, #opts{ event_handlers = EvHandlers, diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 202d8f9373..9a7a384ebb 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -246,22 +246,40 @@ collect_tests_from_file(Specs, Relaxed) -> collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) -> NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes), - catch collect_tests_from_file1(Specs,#testspec{nodes=NodeRefs},Relaxed). - -collect_tests_from_file1([Spec|Specs],TestSpec,Relaxed) -> + %% [Spec1,Spec2,...] means create one testpec record per Spec file + %% [[Spec1,Spec2,...]] means merge all specs into one testspec record + {MergeSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; + true -> {false,Specs} + end, + catch create_specs(Specs1,Specs1,#testspec{nodes=NodeRefs}, + Relaxed,MergeSpecs,[]). + +create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) -> SpecDir = filename:dirname(filename:absname(Spec)), case file:consult(Spec) of {ok,Terms} -> case collect_tests(Terms, TestSpec#testspec{spec_dir=SpecDir}, Relaxed) of - TS = #testspec{tests=Tests, logdir=LogDirs} when Specs == [] -> + TS = #testspec{tests=Tests, logdir=LogDirs} when + Ss == [], MergeSpecs == true -> + LogDirs1 = lists:delete(".",LogDirs) ++ ["."], + [{Specs,TS#testspec{tests=lists:flatten(Tests), + logdir=LogDirs1}}]; + TS = #testspec{tests=Tests, logdir=LogDirs} when + Ss == [], MergeSpecs == false -> LogDirs1 = lists:delete(".",LogDirs) ++ ["."], - TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1}; - TS = #testspec{alias = As, nodes = Ns} -> + TSRet = {[Spec],TS#testspec{tests=lists:flatten(Tests), + logdir=LogDirs1}}, + lists:reverse([TSRet|Saved]); + TS = #testspec{alias = As, nodes = Ns} when + MergeSpecs == true -> TS1 = TS#testspec{alias = lists:reverse(As), nodes = lists:reverse(Ns)}, - collect_tests_from_file1(Specs,TS1,Relaxed) + create_specs(Ss,Specs,TS1,Relaxed,MergeSpecs,[]); + TS when MergeSpecs == false -> + create_specs(Ss,Specs,TestSpec,Relaxed,MergeSpecs, + [{[Spec],TS}|Saved]) end; {error,Reason} -> ReasonStr = -- cgit v1.2.3 From 102c8167c6960233102aa6f2fabec5416882b74f Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Mon, 14 Jan 2013 15:12:01 +0100 Subject: Implement support for including test specifications OTP-9881 --- lib/common_test/src/ct_run.erl | 423 ++++++++++++++++-------------------- lib/common_test/src/ct_testspec.erl | 134 +++++++++--- 2 files changed, 291 insertions(+), 266 deletions(-) (limited to 'lib/common_test/src') diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 50723a763c..742e287493 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -330,33 +330,33 @@ script_start1(Parent, Args) -> true end, - StartOpts = #opts{label = Label, profile = Profile, - vts = Vts, shell = Shell, - cover = Cover, cover_stop = CoverStop, - logdir = LogDir, logopts = LogOpts, - basic_html = BasicHtml, - verbosity = Verbosity, - event_handlers = EvHandlers, - ct_hooks = CTHooks, - enable_builtin_hooks = EnableBuiltinHooks, - auto_compile = AutoCompile, - include = IncludeDirs, - silent_connections = SilentConns, - stylesheet = Stylesheet, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir, - starter = script}, - + Opts = #opts{label = Label, profile = Profile, + vts = Vts, shell = Shell, + cover = Cover, cover_stop = CoverStop, + logdir = LogDir, logopts = LogOpts, + basic_html = BasicHtml, + verbosity = Verbosity, + event_handlers = EvHandlers, + ct_hooks = CTHooks, + enable_builtin_hooks = EnableBuiltinHooks, + auto_compile = AutoCompile, + include = IncludeDirs, + silent_connections = SilentConns, + stylesheet = Stylesheet, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir, + starter = script}, + %% check if log files should be refreshed or go on to run tests... - Result = run_or_refresh(StartOpts, Args), + Result = run_or_refresh(Opts, Args), %% send final results to starting process waiting in script_start/0 Parent ! {self(), Result}. -run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> +run_or_refresh(Opts = #opts{logdir = LogDir}, Args) -> case proplists:get_value(refresh_logs, Args) of undefined -> - script_start2(StartOpts, Args); + script_start2(Opts, Args); Refresh -> LogDir1 = case Refresh of [] -> which(logdir,LogDir); @@ -386,8 +386,8 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> end end. -script_start2(StartOpts = #opts{vts = undefined, - shell = undefined}, Args) -> +script_start2(Opts = #opts{vts = undefined, + shell = undefined}, Args) -> case proplists:get_value(spec, Args) of Specs when Specs =/= [], Specs =/= undefined -> Specs1 = get_start_opt(join_specs, [Specs], Specs, Args), @@ -397,38 +397,38 @@ script_start2(StartOpts = #opts{vts = undefined, {E,Reason} when E == error ; E == 'EXIT' -> {error,Reason}; TestSpecData -> - execute_testspecs(TestSpecData, StartOpts, Args, []) + execute_all_specs(TestSpecData, Opts, Args, []) end; [] -> {error,no_testspec_specified}; _ -> % no testspec used %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), - TheLogDir = which(logdir, StartOpts#opts.logdir), + TheLogDir = which(logdir, Opts#opts.logdir), case check_and_install_configfiles(InitConfig, TheLogDir, - StartOpts) of + Opts) of ok -> % go on read tests from start flags - script_start3(StartOpts#opts{config=InitConfig, - logdir=TheLogDir}, Args); + script_start3(Opts#opts{config=InitConfig, + logdir=TheLogDir}, Args); Error -> Error end end; -script_start2(StartOpts, Args) -> +script_start2(Opts, Args) -> %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), - LogDir = which(logdir, StartOpts#opts.logdir), - case check_and_install_configfiles(InitConfig, LogDir, StartOpts) of + LogDir = which(logdir, Opts#opts.logdir), + case check_and_install_configfiles(InitConfig, LogDir, Opts) of ok -> % go on read tests from start flags - script_start3(StartOpts#opts{config=InitConfig, - logdir=LogDir}, Args); + script_start3(Opts#opts{config=InitConfig, + logdir=LogDir}, Args); Error -> Error end. -execute_testspecs([], _, _, Result) -> +execute_all_specs([], _, _, Result) -> Result1 = lists:reverse(Result), case lists:keysearch('EXIT', 1, Result1) of {value,{_,_,ExitReason}} -> @@ -447,67 +447,93 @@ execute_testspecs([], _, _, Result) -> end end; -execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> - SpecStartOpts = get_data_for_node(TS, node()), - - Label = choose_val(StartOpts#opts.label, - SpecStartOpts#opts.label), - - Profile = choose_val(StartOpts#opts.profile, - SpecStartOpts#opts.profile), - - LogDir = choose_val(StartOpts#opts.logdir, - SpecStartOpts#opts.logdir), - - AllLogOpts = merge_vals([StartOpts#opts.logopts, - SpecStartOpts#opts.logopts]), +execute_all_specs([{Specs,TS} | TSs], Opts, Args, Result) -> + CombinedOpts = combine_test_opts(TS, Specs, Opts), + try execute_one_spec(TS, CombinedOpts, Args) of + ExecResult -> + execute_all_specs(TSs, Opts, Args, [ExecResult|Result]) + catch + _ : ExitReason -> + execute_all_specs(TSs, Opts, Args, + [{'EXIT',self(),ExitReason}|Result]) + end. + +execute_one_spec(TS, Opts, Args) -> + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), + %% merge config from start flags with config from testspec + AllConfig = merge_vals([InitConfig, Opts#opts.config]), + case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of + ok -> % read tests from spec + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir}, Args); + Error -> + Error + end. + +combine_test_opts(TS, Specs, Opts) -> + TSOpts = get_data_for_node(TS, node()), + + Label = choose_val(Opts#opts.label, + TSOpts#opts.label), + + Profile = choose_val(Opts#opts.profile, + TSOpts#opts.profile), + + LogDir = choose_val(Opts#opts.logdir, + TSOpts#opts.logdir), + + AllLogOpts = merge_vals([Opts#opts.logopts, + TSOpts#opts.logopts]), AllVerbosity = - merge_keyvals([StartOpts#opts.verbosity, - SpecStartOpts#opts.verbosity]), + merge_keyvals([Opts#opts.verbosity, + TSOpts#opts.verbosity]), AllSilentConns = - merge_vals([StartOpts#opts.silent_connections, - SpecStartOpts#opts.silent_connections]), + merge_vals([Opts#opts.silent_connections, + TSOpts#opts.silent_connections]), Cover = - choose_val(StartOpts#opts.cover, - SpecStartOpts#opts.cover), + choose_val(Opts#opts.cover, + TSOpts#opts.cover), CoverStop = - choose_val(StartOpts#opts.cover_stop, - SpecStartOpts#opts.cover_stop), + choose_val(Opts#opts.cover_stop, + TSOpts#opts.cover_stop), MultTT = - choose_val(StartOpts#opts.multiply_timetraps, - SpecStartOpts#opts.multiply_timetraps), + choose_val(Opts#opts.multiply_timetraps, + TSOpts#opts.multiply_timetraps), ScaleTT = - choose_val(StartOpts#opts.scale_timetraps, - SpecStartOpts#opts.scale_timetraps), - + choose_val(Opts#opts.scale_timetraps, + TSOpts#opts.scale_timetraps), + CreatePrivDir = - choose_val(StartOpts#opts.create_priv_dir, - SpecStartOpts#opts.create_priv_dir), - + choose_val(Opts#opts.create_priv_dir, + TSOpts#opts.create_priv_dir), + AllEvHs = - merge_vals([StartOpts#opts.event_handlers, - SpecStartOpts#opts.event_handlers]), - + merge_vals([Opts#opts.event_handlers, + TSOpts#opts.event_handlers]), + AllCTHooks = merge_vals( - [StartOpts#opts.ct_hooks, - SpecStartOpts#opts.ct_hooks]), - + [Opts#opts.ct_hooks, + TSOpts#opts.ct_hooks]), + EnableBuiltinHooks = choose_val( - StartOpts#opts.enable_builtin_hooks, - SpecStartOpts#opts.enable_builtin_hooks), - + Opts#opts.enable_builtin_hooks, + TSOpts#opts.enable_builtin_hooks), + Stylesheet = - choose_val(StartOpts#opts.stylesheet, - SpecStartOpts#opts.stylesheet), - - AllInclude = merge_vals([StartOpts#opts.include, - SpecStartOpts#opts.include]), + choose_val(Opts#opts.stylesheet, + TSOpts#opts.stylesheet), + + AllInclude = merge_vals([Opts#opts.include, + TSOpts#opts.include]), application:set_env(common_test, include, AllInclude), - + AutoCompile = - case choose_val(StartOpts#opts.auto_compile, - SpecStartOpts#opts.auto_compile) of + case choose_val(Opts#opts.auto_compile, + TSOpts#opts.auto_compile) of undefined -> true; ACBool -> @@ -518,8 +544,8 @@ execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> end, BasicHtml = - case choose_val(StartOpts#opts.basic_html, - SpecStartOpts#opts.basic_html) of + case choose_val(Opts#opts.basic_html, + TSOpts#opts.basic_html) of undefined -> false; BHBool -> @@ -527,52 +553,27 @@ execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> BHBool), BHBool end, - - Opts = StartOpts#opts{label = Label, - profile = Profile, - testspecs = Specs, - cover = Cover, - cover_stop = CoverStop, - logdir = LogDir, - logopts = AllLogOpts, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = SpecStartOpts#opts.config, - event_handlers = AllEvHs, - ct_hooks = AllCTHooks, - enable_builtin_hooks = - EnableBuiltinHooks, - stylesheet = Stylesheet, - auto_compile = AutoCompile, - include = AllInclude, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir}, - - try execute_testspec(TS, Opts, Args) of - ExecResult -> - execute_testspecs(TSs, StartOpts, Args, [ExecResult | Result]) - catch - _ : ExitReason -> - execute_testspecs(TSs, StartOpts, Args, - [{'EXIT',self(),ExitReason} | Result]) - end. - -execute_testspec(TS, Opts, Args) -> - %% read config/userconfig from start flags - InitConfig = ct_config:prepare_config_list(Args), - TheLogDir = which(logdir, Opts#opts.logdir), - %% merge config from start flags with config from testspec - AllConfig = merge_vals([InitConfig, Opts#opts.config]), - case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of - ok -> % read tests from spec - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig, - logdir=TheLogDir}, Args); - Error -> - Error - end. + + Opts#opts{label = Label, + profile = Profile, + testspecs = Specs, + cover = Cover, + cover_stop = CoverStop, + logdir = which(logdir, LogDir), + logopts = AllLogOpts, + basic_html = BasicHtml, + verbosity = AllVerbosity, + silent_connections = AllSilentConns, + config = TSOpts#opts.config, + event_handlers = AllEvHs, + ct_hooks = AllCTHooks, + enable_builtin_hooks = EnableBuiltinHooks, + stylesheet = Stylesheet, + auto_compile = AutoCompile, + include = AllInclude, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir}. check_and_install_configfiles( Configs, LogDir, #opts{ @@ -593,12 +594,12 @@ check_and_install_configfiles( {error,{cant_load_callback_module,Info}} end. -script_start3(StartOpts, Args) -> - StartOpts1 = get_start_opt(step, - fun(Step) -> - StartOpts#opts{step = Step, - cover = undefined} - end, StartOpts, Args), +script_start3(Opts, Args) -> + Opts1 = get_start_opt(step, + fun(Step) -> + Opts#opts{step = Step, + cover = undefined} + end, Opts, Args), case {proplists:get_value(dir, Args), proplists:get_value(suite, Args), groups_and_cases(proplists:get_value(group, Args), @@ -612,17 +613,17 @@ script_start3(StartOpts, Args) -> {error,no_dir_specified}; {Dirs,undefined,[]} when is_list(Dirs) -> - script_start4(StartOpts#opts{tests = tests(Dirs)}, Args); + script_start4(Opts#opts{tests = tests(Dirs)}, Args); {undefined,Suites,[]} when is_list(Suites) -> Ts = tests([suite_to_test(S) || S <- Suites]), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); {undefined,Suite,GsAndCs} when is_list(Suite) -> case [suite_to_test(S) || S <- Suite] of DirMods = [_] -> Ts = tests(DirMods, GsAndCs), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); [_,_|_] -> {error,multiple_suites_and_cases}; _ -> @@ -636,10 +637,10 @@ script_start3(StartOpts, Args) -> case [suite_to_test(Dir,S) || S <- Suite] of DirMods when GsAndCs == [] -> Ts = tests(DirMods), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); DirMods = [_] when GsAndCs /= [] -> Ts = tests(DirMods, GsAndCs), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); [_,_|_] when GsAndCs /= [] -> {error,multiple_suites_and_cases}; _ -> @@ -650,8 +651,8 @@ script_start3(StartOpts, Args) -> {error,incorrect_start_options}; {undefined,undefined,_} -> - if StartOpts#opts.vts ; StartOpts#opts.shell -> - script_start4(StartOpts#opts{tests = []}, Args); + if Opts#opts.vts ; Opts#opts.shell -> + script_start4(Opts#opts{tests = []}, Args); true -> script_usage(), {error,missing_start_options} @@ -781,6 +782,7 @@ script_usage() -> "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" "\n\t[-allow_user_terms]" + "\n\t[-join_specs]" "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" @@ -1075,105 +1077,60 @@ run_test2(StartOpts) -> end. run_spec_file(Relaxed, - Opts = #opts{testspecs = Specs, config = CfgFiles}, + Opts = #opts{testspecs = Specs}, StartOpts) -> Specs1 = case Specs of [X|_] when is_integer(X) -> [Specs]; _ -> Specs end, AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), - log_ts_names(AbsSpecs), - case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of + AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts), + case catch ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of {Error,CTReason} when Error == error ; Error == 'EXIT' -> exit({error,CTReason}); - TS -> - SpecOpts = get_data_for_node(TS, node()), - Label = choose_val(Opts#opts.label, - SpecOpts#opts.label), - Profile = choose_val(Opts#opts.profile, - SpecOpts#opts.profile), - LogDir = choose_val(Opts#opts.logdir, - SpecOpts#opts.logdir), - AllLogOpts = merge_vals([Opts#opts.logopts, - SpecOpts#opts.logopts]), - Stylesheet = choose_val(Opts#opts.stylesheet, - SpecOpts#opts.stylesheet), - AllVerbosity = merge_keyvals([Opts#opts.verbosity, - SpecOpts#opts.verbosity]), - AllSilentConns = merge_vals([Opts#opts.silent_connections, - SpecOpts#opts.silent_connections]), - AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), - Cover = choose_val(Opts#opts.cover, - SpecOpts#opts.cover), - CoverStop = choose_val(Opts#opts.cover_stop, - SpecOpts#opts.cover_stop), - MultTT = choose_val(Opts#opts.multiply_timetraps, - SpecOpts#opts.multiply_timetraps), - ScaleTT = choose_val(Opts#opts.scale_timetraps, - SpecOpts#opts.scale_timetraps), - CreatePrivDir = choose_val(Opts#opts.create_priv_dir, - SpecOpts#opts.create_priv_dir), - AllEvHs = merge_vals([Opts#opts.event_handlers, - SpecOpts#opts.event_handlers]), - AllInclude = merge_vals([Opts#opts.include, - SpecOpts#opts.include]), - AllCTHooks = merge_vals([Opts#opts.ct_hooks, - SpecOpts#opts.ct_hooks]), - EnableBuiltinHooks = choose_val(Opts#opts.enable_builtin_hooks, - SpecOpts#opts.enable_builtin_hooks), - - application:set_env(common_test, include, AllInclude), - - AutoCompile = case choose_val(Opts#opts.auto_compile, - SpecOpts#opts.auto_compile) of - undefined -> - true; - ACBool -> - application:set_env(common_test, auto_compile, - ACBool), - ACBool - end, + TestSpecData -> + run_all_specs(TestSpecData, Opts, StartOpts, []) + end. - BasicHtml = case choose_val(Opts#opts.basic_html, - SpecOpts#opts.basic_html) of - undefined -> - false; - BHBool -> - application:set_env(common_test, basic_html, - BHBool), - BHBool - end, - - Opts1 = Opts#opts{label = Label, - profile = Profile, - cover = Cover, - cover_stop = CoverStop, - logdir = which(logdir, LogDir), - logopts = AllLogOpts, - stylesheet = Stylesheet, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = AllConfig, - event_handlers = AllEvHs, - auto_compile = AutoCompile, - include = AllInclude, - testspecs = AbsSpecs, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir, - ct_hooks = AllCTHooks, - enable_builtin_hooks = EnableBuiltinHooks - }, - - case check_and_install_configfiles(AllConfig,Opts1#opts.logdir, - Opts1) of - ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); - {error,GCFReason} -> - exit({error,GCFReason}) +run_all_specs([], _, _, TotResult) -> + TotResult1 = lists:reverse(TotResult), + case lists:keysearch('EXIT', 1, TotResult1) of + {value,{_,_,ExitReason}} -> + exit(ExitReason); + false -> + case lists:keysearch(error, 1, TotResult1) of + {value,Error} -> + Error; + false -> + lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, + {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> + {Ok1+Ok,Fail1+Fail, + {UserSkip1+UserSkip, + AutoSkip1+AutoSkip}} + end, {0,0,{0,0}}, TotResult1) end + end; + +run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) -> + log_ts_names(Specs), + Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts), + AllConfig = merge_vals([Opts#opts.config, TSConfig]), + try run_one_spec(TS, Combined#opts{config = AllConfig}, StartOpts) of + Result -> + run_all_specs(TSs, Opts, StartOpts, [Result | TotResult]) + catch + _ : Reason -> + run_all_specs(TSs, Opts, StartOpts, [{error,Reason} | TotResult]) + end. + +run_one_spec(TS, CombinedOpts, StartOpts) -> + #opts{logdir = Logdir, config = Config} = CombinedOpts, + case check_and_install_configfiles(Config, Logdir, CombinedOpts) of + ok -> + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + reformat_result(catch do_run(Run, Skip, CombinedOpts, StartOpts)); + Error -> + Error end. run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, @@ -2927,6 +2884,10 @@ opts2args(EnvStartOpts) -> [{allow_user_terms,[]}]; ({allow_user_terms,false}) -> []; + ({join_specs,true}) -> + [{join_specs,[]}]; + ({join_specs,false}) -> + []; ({auto_compile,false}) -> [{no_auto_compile,[]}]; ({auto_compile,true}) -> diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 9a7a384ebb..3d9351b71b 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -241,45 +241,71 @@ get_skipped_cases1(_,_,_,[]) -> %%% collect_tests_from_file reads a testspec file and returns a record %%% containing the data found. -collect_tests_from_file(Specs, Relaxed) -> +collect_tests_from_file(Specs,Relaxed) -> collect_tests_from_file(Specs,[node()],Relaxed). collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) -> NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes), %% [Spec1,Spec2,...] means create one testpec record per Spec file %% [[Spec1,Spec2,...]] means merge all specs into one testspec record - {MergeSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; + {JoinSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; true -> {false,Specs} end, - catch create_specs(Specs1,Specs1,#testspec{nodes=NodeRefs}, - Relaxed,MergeSpecs,[]). + TS0 = #testspec{nodes=NodeRefs}, + %% remove specs without tests + Filter = fun({_,#testspec{tests=[]}}) -> false; + (_) -> true + end, + try create_specs(Specs1,TS0,Relaxed,JoinSpecs,{[],TS0},[]) of + {{[],_},AdditionalTestSpecs} -> + lists:filter(Filter,AdditionalTestSpecs); + {{_,#testspec{tests=[]}},AdditionalTestSpecs} -> + lists:filter(Filter,AdditionalTestSpecs); + {{JoinedSpecs,JoinedTestSpec},AdditionalTestSpecs} -> + [{JoinedSpecs,JoinedTestSpec} | + lists:filter(Filter,AdditionalTestSpecs)] + catch + _:Error -> + Error + end. -create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) -> +create_specs([],_,_,_,Joined,Additional) -> + {Joined,Additional}; +create_specs([Spec|Ss],TestSpec,Relaxed,JoinSpecs, + Joined={JSpecs,_},Additional) -> SpecDir = filename:dirname(filename:absname(Spec)), + TestSpec1 = TestSpec#testspec{spec_dir=SpecDir}, case file:consult(Spec) of - {ok,Terms} -> - case collect_tests(Terms, - TestSpec#testspec{spec_dir=SpecDir}, - Relaxed) of - TS = #testspec{tests=Tests, logdir=LogDirs} when - Ss == [], MergeSpecs == true -> - LogDirs1 = lists:delete(".",LogDirs) ++ ["."], - [{Specs,TS#testspec{tests=lists:flatten(Tests), - logdir=LogDirs1}}]; - TS = #testspec{tests=Tests, logdir=LogDirs} when - Ss == [], MergeSpecs == false -> - LogDirs1 = lists:delete(".",LogDirs) ++ ["."], - TSRet = {[Spec],TS#testspec{tests=lists:flatten(Tests), - logdir=LogDirs1}}, - lists:reverse([TSRet|Saved]); - TS = #testspec{alias = As, nodes = Ns} when - MergeSpecs == true -> - TS1 = TS#testspec{alias = lists:reverse(As), - nodes = lists:reverse(Ns)}, - create_specs(Ss,Specs,TS1,Relaxed,MergeSpecs,[]); - TS when MergeSpecs == false -> - create_specs(Ss,Specs,TestSpec,Relaxed,MergeSpecs, - [{[Spec],TS}|Saved]) + {ok,Terms} -> + Terms1 = replace_names(Terms), + {Specs2Join,Specs2Add} = get_included_specs(Terms1,TestSpec1), + TestSpec2 = create_spec(Terms1,TestSpec1, + Relaxed,JoinSpecs), + case {JoinSpecs,Specs2Join,Specs2Add} of + {true,[],[]} -> + create_specs(Ss,TestSpec2,Relaxed,JoinSpecs, + {JSpecs++[get_absdir(Spec,TestSpec2)], + TestSpec2},Additional); + {false,[],[]} -> + create_specs(Ss,TestSpec,Relaxed,JoinSpecs,Joined, + Additional++[{[get_absdir(Spec,TestSpec2)], + TestSpec2}]); + _ -> + {{JSpecs1,JTS1},Additional1} = + create_specs(Specs2Join,TestSpec2,Relaxed,true, + {[get_absdir(Spec,TestSpec2)], + TestSpec2},[]), + {Joined2,Additional2} = + create_specs(Specs2Add,TestSpec,Relaxed,false, + {[],TestSpec1},[]), + NewJoined = {JSpecs++JSpecs1,JTS1}, + NewAdditional = Additional++Additional1++ + [Joined2 | Additional2], + NextTestSpec = if not JoinSpecs -> TestSpec; + true -> JTS1 + end, + create_specs(Ss,NextTestSpec,Relaxed,JoinSpecs, + NewJoined,NewAdditional) end; {error,Reason} -> ReasonStr = @@ -287,6 +313,14 @@ create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) -> [file:format_error(Reason)])), throw({error,{Spec,ReasonStr}}) end. + +create_spec(Terms,TestSpec,Relaxed,JoinSpecs) -> + TS = #testspec{tests=Tests, logdir=LogDirs} = + collect_tests({false,Terms},TestSpec,Relaxed), + + LogDirs1 = lists:delete(".",LogDirs) ++ ["."], + TS#testspec{tests=lists:flatten(Tests), + logdir=LogDirs1}. collect_tests_from_list(Terms,Relaxed) -> collect_tests_from_list(Terms,[node()],Relaxed). @@ -294,8 +328,8 @@ collect_tests_from_list(Terms,Relaxed) -> collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) -> {ok,Cwd} = file:get_cwd(), NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes), - case catch collect_tests(Terms,#testspec{nodes=NodeRefs, - spec_dir=Cwd}, + case catch collect_tests({true,Terms},#testspec{nodes=NodeRefs, + spec_dir=Cwd}, Relaxed) of E = {error,_} -> E; @@ -305,10 +339,15 @@ collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) -> TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1} end. -collect_tests(Terms,TestSpec,Relaxed) -> +collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) -> put(relaxed,Relaxed), - Terms1 = replace_names(Terms), - TestSpec1 = get_global(Terms1,TestSpec), + Terms1 = if Replace -> replace_names(Terms); + true -> Terms + end, + %% reverse nodes and aliases initially to get the order of them right + %% in case this spec is being joined with a previous one + TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As), + nodes = lists:reverse(Ns)}), TestSpec2 = get_all_nodes(Terms1,TestSpec1), {Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2), add_tests(Terms2,TestSpec3). @@ -438,9 +477,30 @@ replace_names_in_node1(NodeStr,Defs=[{Name,Replacement}|Ds]) -> replace_names_in_node1(NodeStr,[]) -> NodeStr. +%% look for other specification files, either to join with the +%% current spec, or execute as additional test runs +get_included_specs(Terms,TestSpec) -> + get_included_specs(Terms,TestSpec,[],[]). + +get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Add) -> + Specs = case SpecOrSpecs of + [File|_] when is_list(File) -> + [get_absdir(Spec,TestSpec) || Spec <- SpecOrSpecs]; + [Ch|_] when is_integer(Ch) -> + [get_absdir(SpecOrSpecs,TestSpec)] + end, + if How == join -> + get_included_specs(Ts,TestSpec,Join++Specs,Add); + true -> + get_included_specs(Ts,TestSpec,Join,Add++Specs) + end; +get_included_specs([_|Ts],TestSpec,Join,Add) -> + get_included_specs(Ts,TestSpec,Join,Add); +get_included_specs([],_,Join,Add) -> + {Join,Add}. %% global terms that will be used for analysing all other terms in the spec -get_global([{merge_tests,Bool} | Ts], Spec) -> +get_global([{merge_tests,Bool}|Ts],Spec) -> get_global(Ts,Spec#testspec{merge_tests=Bool}); %% the 'define' term replaces the 'alias' and 'node' terms, but we need to keep @@ -810,7 +870,10 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled add_tests([{node,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); -add_tests([{merge_tests, _} | Ts], Spec) -> % handled +add_tests([{merge_tests,_} | Ts], Spec) -> % handled + add_tests(Ts,Spec); + +add_tests([{specs,_,_} | Ts], Spec) -> % handled add_tests(Ts,Spec); %% -------------------------------------------------- @@ -1279,6 +1342,7 @@ is_node([],_) -> valid_terms() -> [ {define,3}, + {specs,3}, {node,3}, {cover,2}, {cover,3}, -- cgit v1.2.3 From 6e8de699f548762e2e6894d347e9dc285db11841 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Mon, 21 Jan 2013 17:14:55 +0100 Subject: Update CT Master OTP-9881 --- lib/common_test/src/ct_master.erl | 158 +++++++++++++++++++++--------------- lib/common_test/src/ct_testspec.erl | 88 ++++++++++---------- 2 files changed, 136 insertions(+), 110 deletions(-) (limited to 'lib/common_test/src') diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index f29eba605c..d58395b692 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -82,39 +82,48 @@ run_test(NodeOptsList) when is_list(NodeOptsList) -> %%% ExclNodes = [atom()] %%% %%% @doc Tests are spawned on the nodes as specified in TestSpecs. -%%% Each specification in TestSpec will be handled separately. It is however possible -%%% to also specify a list of specifications that should be merged into one before -%%% the tests are executed. Any test without a particular node specification will -%%% also be executed on the nodes in InclNodes. Nodes in the -%%% ExclNodes list will be excluded from the test. +%%% Each specification in TestSpec will be handled separately. It is however +%%% possible to also specify a list of specifications that should be merged +%%% into one before the tests are executed. Any test without a particular node +%%% specification will also be executed on the nodes in InclNodes. +%%% Nodes in the ExclNodes list will be excluded from the test. run([TS|TestSpecs],AllowUserTerms,InclNodes,ExclNodes) when is_list(TS), is_list(InclNodes), is_list(ExclNodes) -> - TS1 = - case TS of - List=[S|_] when is_list(S) -> List; - Spec -> [Spec] - end, - Result = - case catch ct_testspec:collect_tests_from_file(TS1,InclNodes,AllowUserTerms) of - {error,Reason} -> - {error,Reason}; - TSRec=#testspec{logdir=AllLogDirs, - config=StdCfgFiles, - userconfig=UserCfgFiles, - include=AllIncludes, - init=AllInitOpts, - event_handler=AllEvHs} -> - AllCfgFiles = {StdCfgFiles, UserCfgFiles}, - RunSkipPerNode = ct_testspec:prepare_tests(TSRec), - RunSkipPerNode2 = exclude_nodes(ExclNodes,RunSkipPerNode), - run_all(RunSkipPerNode2,AllLogDirs,AllCfgFiles,AllEvHs, - AllIncludes,[],[],AllInitOpts,TS1) - end, - [{TS,Result} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)]; + %% Note: [Spec] means run one test with Spec + %% [Spec1,Spec2] means run two tests separately + %% [[Spec1,Spec2]] means run one test, with the two specs merged + case catch ct_testspec:collect_tests_from_file([TS],InclNodes, + AllowUserTerms) of + {error,Reason} -> + [{error,Reason} | run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)]; + Tests -> + RunResult = + lists:map( + fun({Specs,TSRec=#testspec{logdir=AllLogDirs, + config=StdCfgFiles, + userconfig=UserCfgFiles, + include=AllIncludes, + init=AllInitOpts, + event_handler=AllEvHs}}) -> + AllCfgFiles = + {StdCfgFiles,UserCfgFiles}, + RunSkipPerNode = + ct_testspec:prepare_tests(TSRec), + RunSkipPerNode2 = + exclude_nodes(ExclNodes,RunSkipPerNode), + TSList = if is_integer(hd(TS)) -> [TS]; + true -> TS end, + {Specs,run_all(RunSkipPerNode2,AllLogDirs, + AllCfgFiles,AllEvHs, + AllIncludes,[],[],AllInitOpts,TSList)} + end, Tests), + RunResult ++ run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes) + end; run([],_,_,_) -> []; -run(TS,AllowUserTerms,InclNodes,ExclNodes) when is_list(InclNodes), is_list(ExclNodes) -> +run(TS,AllowUserTerms,InclNodes,ExclNodes) when is_list(InclNodes), + is_list(ExclNodes) -> run([TS],AllowUserTerms,InclNodes,ExclNodes). %%%----------------------------------------------------------------- @@ -152,29 +161,32 @@ exclude_nodes([],RunSkipPerNode) -> %%% AllowUserTerms = bool() %%% Node = atom() %%% -%%% @doc Tests are spawned on Node according to TestSpecs. +%%% @doc Tests are spawned on Node according to +%%% TestSpecs. run_on_node([TS|TestSpecs],AllowUserTerms,Node) when is_list(TS),is_atom(Node) -> - TS1 = - case TS of - [List|_] when is_list(List) -> List; - Spec -> [Spec] - end, - Result = - case catch ct_testspec:collect_tests_from_file(TS1,[Node],AllowUserTerms) of - {error,Reason} -> - {error,Reason}; - TSRec=#testspec{logdir=AllLogDirs, - config=StdCfgFiles, - init=AllInitOpts, - include=AllIncludes, - userconfig=UserCfgFiles, - event_handler=AllEvHs} -> - AllCfgFiles = {StdCfgFiles, UserCfgFiles}, - {Run,Skip} = ct_testspec:prepare_tests(TSRec,Node), - run_all([{Node,Run,Skip}],AllLogDirs,AllCfgFiles,AllEvHs, - AllIncludes, [],[],AllInitOpts,TS1) - end, - [{TS,Result} | run_on_node(TestSpecs,AllowUserTerms,Node)]; + case catch ct_testspec:collect_tests_from_file([TS],[Node], + AllowUserTerms) of + {error,Reason} -> + [{error,Reason} | run_on_node(TestSpecs,AllowUserTerms,Node)]; + Tests -> + RunResult = + lists:map( + fun({Specs,TSRec=#testspec{logdir=AllLogDirs, + config=StdCfgFiles, + init=AllInitOpts, + include=AllIncludes, + userconfig=UserCfgFiles, + event_handler=AllEvHs}}) -> + AllCfgFiles = {StdCfgFiles,UserCfgFiles}, + {Run,Skip} = ct_testspec:prepare_tests(TSRec,Node), + TSList = if is_integer(hd(TS)) -> [TS]; + true -> TS end, + {Specs,run_all([{Node,Run,Skip}],AllLogDirs, + AllCfgFiles,AllEvHs, + AllIncludes, [],[],AllInitOpts,TSList)} + end, Tests), + RunResult ++ run_on_node(TestSpecs,AllowUserTerms,Node) + end; run_on_node([],_,_) -> []; run_on_node(TS,AllowUserTerms,Node) when is_atom(Node) -> @@ -245,7 +257,8 @@ run_all([],AllLogDirs,_,AllEvHs,_AllIncludes, false -> "." end, log(tty,"Master Logdir","~s",[MasterLogDir]), - start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,LogDirs,InitOptions,Specs), + start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir, + LogDirs,InitOptions,Specs), ok. @@ -297,13 +310,15 @@ start_master(NodeOptsList) -> start_master(NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Specs) -> Master = spawn_link(?MODULE,init_master,[self(),NodeOptsList,EvHandlers, - MasterLogDir,LogDirs,InitOptions,Specs]), + MasterLogDir,LogDirs, + InitOptions,Specs]), receive {Master,Result} -> Result end. %%% @hidden -init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Specs) -> +init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs, + InitOptions,Specs) -> case whereis(ct_master) of undefined -> register(ct_master,self()), @@ -325,6 +340,7 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Spec {MLPid,_} = ct_master_logs:start(MasterLogDir, [N || {N,_} <- NodeOptsList]), log(all,"Master Logger process started","~w",[MLPid]), + case Specs of [] -> ok; _ -> @@ -359,7 +375,8 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,InitOptions,Spec init_master1(Parent,NodeOptsList,InitOptions,LogDirs). init_master1(Parent,NodeOptsList,InitOptions,LogDirs) -> - {Inaccessible,NodeOptsList1,InitOptions1} = init_nodes(NodeOptsList,InitOptions), + {Inaccessible,NodeOptsList1,InitOptions1} = init_nodes(NodeOptsList, + InitOptions), case Inaccessible of [] -> init_master2(Parent,NodeOptsList,LogDirs); @@ -392,7 +409,8 @@ init_master2(Parent,NodeOptsList,LogDirs) -> fun({Node,Opts}) -> monitor_node(Node,true), log(all,"Test Info","Starting test(s) on ~p...",[Node]), - {spawn_link(Node,?MODULE,init_node_ctrl,[self(),Cookie,Opts]),Node} + {spawn_link(Node,?MODULE,init_node_ctrl,[self(),Cookie,Opts]), + Node} end, NodeCtrlPids = lists:map(SpawnAndMon,NodeOptsList), Result = master_loop(#state{node_ctrl_pids=NodeCtrlPids, @@ -404,7 +422,8 @@ master_loop(#state{node_ctrl_pids=[], results=Finished}) -> Str = lists:map(fun({Node,Result}) -> - io_lib:format("~-40.40.*s~p\n",[$_,atom_to_list(Node),Result]) + io_lib:format("~-40.40.*s~p\n", + [$_,atom_to_list(Node),Result]) end,lists:reverse(Finished)), log(all,"TEST RESULTS",Str,[]), log(all,"Info","Updating log files",[]), @@ -437,11 +456,13 @@ master_loop(State=#state{node_ctrl_pids=NodeCtrlPids, Bad end, log(all,"Test Info", - "Test on node ~w failed! Reason: ~p",[Node,Error]), + "Test on node ~w failed! Reason: ~p", + [Node,Error]), {Locks1,Blocked1} = update_queue(exit,Node,Locks,Blocked), master_loop(State#state{node_ctrl_pids=NodeCtrlPids1, - results=[{Node,Error}|Results], + results=[{Node, + Error}|Results], locks=Locks1, blocked=Blocked1}) end; @@ -752,7 +773,8 @@ start_nodes(InitOptions)-> IsAlive = lists:member(NodeName, nodes()), case {HasNodeStart, IsAlive} of {false, false}-> - io:format("WARNING: Node ~p is not alive but has no node_start option~n", [NodeName]); + io:format("WARNING: Node ~p is not alive but has no " + "node_start option~n", [NodeName]); {false, true}-> io:format("Node ~p is alive~n", [NodeName]); {true, false}-> @@ -761,12 +783,15 @@ start_nodes(InitOptions)-> lists:keytake(callback_module, 1, NodeStart), case Callback:start(Host, Node, NodeStart2) of {ok, NodeName} -> - io:format("Node ~p started successfully with callback ~p~n", [NodeName,Callback]); + io:format("Node ~p started successfully " + "with callback ~p~n", [NodeName,Callback]); {error, Reason, _NodeName} -> - io:format("Failed to start node ~p with callback ~p! Reason: ~p~n", [NodeName, Callback, Reason]) + io:format("Failed to start node ~p with callback ~p! " + "Reason: ~p~n", [NodeName, Callback, Reason]) end; {true, true}-> - io:format("WARNING: Node ~p is alive but has node_start option~n", [NodeName]) + io:format("WARNING: Node ~p is alive but has node_start " + "option~n", [NodeName]) end end, InitOptions). @@ -779,7 +804,8 @@ eval_on_nodes(InitOptions)-> {false,_}-> ok; {true,false}-> - io:format("WARNING: Node ~p is not alive but has eval option ~n", [NodeName]); + io:format("WARNING: Node ~p is not alive but has eval " + "option ~n", [NodeName]); {true,true}-> {eval, MFAs} = lists:keyfind(eval, 1, Options), evaluate(NodeName, MFAs) @@ -790,9 +816,11 @@ eval_on_nodes(InitOptions)-> evaluate(Node, [{M,F,A}|MFAs])-> case rpc:call(Node, M, F, A) of {badrpc,Reason}-> - io:format("WARNING: Failed to call ~p:~p/~p on node ~p due to ~p~n", [M,F,length(A),Node,Reason]); + io:format("WARNING: Failed to call ~p:~p/~p on node ~p " + "due to ~p~n", [M,F,length(A),Node,Reason]); Result-> - io:format("Called ~p:~p/~p on node ~p, result: ~p~n", [M,F,length(A),Node,Result]) + io:format("Called ~p:~p/~p on node ~p, result: ~p~n", + [M,F,length(A),Node,Result]) end, evaluate(Node, MFAs); evaluate(_Node, [])-> diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 3d9351b71b..e698f13b9a 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -257,55 +257,54 @@ collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) -> (_) -> true end, try create_specs(Specs1,TS0,Relaxed,JoinSpecs,{[],TS0},[]) of - {{[],_},AdditionalTestSpecs} -> - lists:filter(Filter,AdditionalTestSpecs); - {{_,#testspec{tests=[]}},AdditionalTestSpecs} -> - lists:filter(Filter,AdditionalTestSpecs); - {{JoinedSpecs,JoinedTestSpec},AdditionalTestSpecs} -> + {{[],_},SeparateTestSpecs} -> + lists:filter(Filter,SeparateTestSpecs); + {{_,#testspec{tests=[]}},SeparateTestSpecs} -> + lists:filter(Filter,SeparateTestSpecs); + {{JoinedSpecs,JoinedTestSpec},SeparateTestSpecs} -> [{JoinedSpecs,JoinedTestSpec} | - lists:filter(Filter,AdditionalTestSpecs)] + lists:filter(Filter,SeparateTestSpecs)] catch _:Error -> Error end. -create_specs([],_,_,_,Joined,Additional) -> - {Joined,Additional}; +create_specs([],_,_,_,Joined,Separate) -> + {Joined,Separate}; create_specs([Spec|Ss],TestSpec,Relaxed,JoinSpecs, - Joined={JSpecs,_},Additional) -> + Joined={JSpecs,_},Separate) -> SpecDir = filename:dirname(filename:absname(Spec)), TestSpec1 = TestSpec#testspec{spec_dir=SpecDir}, case file:consult(Spec) of {ok,Terms} -> Terms1 = replace_names(Terms), - {Specs2Join,Specs2Add} = get_included_specs(Terms1,TestSpec1), - TestSpec2 = create_spec(Terms1,TestSpec1, - Relaxed,JoinSpecs), - case {JoinSpecs,Specs2Join,Specs2Add} of + {Specs2Join,SepSpecs} = get_included_specs(Terms1,TestSpec1), + TestSpec2 = create_spec(Terms1,TestSpec1,Relaxed), + case {JoinSpecs,Specs2Join,SepSpecs} of {true,[],[]} -> create_specs(Ss,TestSpec2,Relaxed,JoinSpecs, {JSpecs++[get_absdir(Spec,TestSpec2)], - TestSpec2},Additional); + TestSpec2},Separate); {false,[],[]} -> create_specs(Ss,TestSpec,Relaxed,JoinSpecs,Joined, - Additional++[{[get_absdir(Spec,TestSpec2)], + Separate++[{[get_absdir(Spec,TestSpec2)], TestSpec2}]); _ -> - {{JSpecs1,JTS1},Additional1} = + {{JSpecs1,JTS1},Separate1} = create_specs(Specs2Join,TestSpec2,Relaxed,true, {[get_absdir(Spec,TestSpec2)], TestSpec2},[]), - {Joined2,Additional2} = - create_specs(Specs2Add,TestSpec,Relaxed,false, + {Joined2,Separate2} = + create_specs(SepSpecs,TestSpec,Relaxed,false, {[],TestSpec1},[]), NewJoined = {JSpecs++JSpecs1,JTS1}, - NewAdditional = Additional++Additional1++ - [Joined2 | Additional2], + NewSeparate = Separate++Separate1++ + [Joined2 | Separate2], NextTestSpec = if not JoinSpecs -> TestSpec; true -> JTS1 end, create_specs(Ss,NextTestSpec,Relaxed,JoinSpecs, - NewJoined,NewAdditional) + NewJoined,NewSeparate) end; {error,Reason} -> ReasonStr = @@ -314,10 +313,9 @@ create_specs([Spec|Ss],TestSpec,Relaxed,JoinSpecs, throw({error,{Spec,ReasonStr}}) end. -create_spec(Terms,TestSpec,Relaxed,JoinSpecs) -> +create_spec(Terms,TestSpec,Relaxed) -> TS = #testspec{tests=Tests, logdir=LogDirs} = collect_tests({false,Terms},TestSpec,Relaxed), - LogDirs1 = lists:delete(".",LogDirs) ++ ["."], TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1}. @@ -478,11 +476,11 @@ replace_names_in_node1(NodeStr,[]) -> NodeStr. %% look for other specification files, either to join with the -%% current spec, or execute as additional test runs +%% current spec, or execute as separate test runs get_included_specs(Terms,TestSpec) -> get_included_specs(Terms,TestSpec,[],[]). -get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Add) -> +get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Sep) -> Specs = case SpecOrSpecs of [File|_] when is_list(File) -> [get_absdir(Spec,TestSpec) || Spec <- SpecOrSpecs]; @@ -490,14 +488,14 @@ get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Add) -> [get_absdir(SpecOrSpecs,TestSpec)] end, if How == join -> - get_included_specs(Ts,TestSpec,Join++Specs,Add); + get_included_specs(Ts,TestSpec,Join++Specs,Sep); true -> - get_included_specs(Ts,TestSpec,Join,Add++Specs) + get_included_specs(Ts,TestSpec,Join,Sep++Specs) end; -get_included_specs([_|Ts],TestSpec,Join,Add) -> - get_included_specs(Ts,TestSpec,Join,Add); -get_included_specs([],_,Join,Add) -> - {Join,Add}. +get_included_specs([_|Ts],TestSpec,Join,Sep) -> + get_included_specs(Ts,TestSpec,Join,Sep); +get_included_specs([],_,Join,Sep) -> + {Join,Sep}. %% global terms that will be used for analysing all other terms in the spec get_global([{merge_tests,Bool}|Ts],Spec) -> @@ -715,7 +713,7 @@ add_tests([{suites,all_nodes,Dir,Ss}|Ts],Spec) -> add_tests([{suites,Dir,Ss}|Ts],Spec) -> add_tests([{suites,all_nodes,Dir,Ss}|Ts],Spec); add_tests([{suites,Nodes,Dir,Ss}|Ts],Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,suites,[Dir,Ss],Ts,Spec#testspec.nodes), + Ts1 = per_node(Nodes,suites,[Dir,Ss],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{suites,Node,Dir,Ss}|Ts],Spec) -> Tests = Spec#testspec.tests, @@ -738,11 +736,11 @@ add_tests([{groups,Dir,Suite,Gs}|Ts],Spec) -> add_tests([{groups,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) -> add_tests([{groups,all_nodes,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec); add_tests([{groups,Nodes,Dir,Suite,Gs}|Ts],Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,groups,[Dir,Suite,Gs],Ts,Spec#testspec.nodes), + Ts1 = per_node(Nodes,groups,[Dir,Suite,Gs],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{groups,Nodes,Dir,Suite,Gs,{cases,TCs}}|Ts], Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,groups,[Dir,Suite,Gs,{cases,TCs}],Ts, + Ts1 = per_node(Nodes,groups,[Dir,Suite,Gs,{cases,TCs}],Ts, Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) -> @@ -766,7 +764,7 @@ add_tests([{cases,all_nodes,Dir,Suite,Cs}|Ts],Spec) -> add_tests([{cases,Dir,Suite,Cs}|Ts],Spec) -> add_tests([{cases,all_nodes,Dir,Suite,Cs}|Ts],Spec); add_tests([{cases,Nodes,Dir,Suite,Cs}|Ts],Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,cases,[Dir,Suite,Cs],Ts,Spec#testspec.nodes), + Ts1 = per_node(Nodes,cases,[Dir,Suite,Cs],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) -> Tests = Spec#testspec.tests, @@ -781,7 +779,7 @@ add_tests([{skip_suites,all_nodes,Dir,Ss,Cmt}|Ts],Spec) -> add_tests([{skip_suites,Dir,Ss,Cmt}|Ts],Spec) -> add_tests([{skip_suites,all_nodes,Dir,Ss,Cmt}|Ts],Spec); add_tests([{skip_suites,Nodes,Dir,Ss,Cmt}|Ts],Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,skip_suites,[Dir,Ss,Cmt],Ts,Spec#testspec.nodes), + Ts1 = per_node(Nodes,skip_suites,[Dir,Ss,Cmt],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, @@ -802,11 +800,11 @@ add_tests([{skip_groups,Dir,Suite,Gs,Cmt}|Ts],Spec) -> add_tests([{skip_groups,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) -> add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec); add_tests([{skip_groups,Nodes,Dir,Suite,Gs,Cmt}|Ts],Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,Cmt],Ts,Spec#testspec.nodes), + Ts1 = per_node(Nodes,skip_groups,[Dir,Suite,Gs,Cmt],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{skip_groups,Nodes,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts], Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,{cases,TCs},Cmt],Ts, + Ts1 = per_node(Nodes,skip_groups,[Dir,Suite,Gs,{cases,TCs},Cmt],Ts, Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) -> @@ -830,7 +828,7 @@ add_tests([{skip_cases,all_nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) -> add_tests([{skip_cases,Dir,Suite,Cs,Cmt}|Ts],Spec) -> add_tests([{skip_cases,all_nodes,Dir,Suite,Cs,Cmt}|Ts],Spec); add_tests([{skip_cases,Nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) when is_list(Nodes) -> - Ts1 = separate(Nodes,skip_cases,[Dir,Suite,Cs,Cmt],Ts,Spec#testspec.nodes), + Ts1 = per_node(Nodes,skip_cases,[Dir,Suite,Cs,Cmt],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) -> Tests = Spec#testspec.tests, @@ -902,7 +900,7 @@ add_tests([{Tag,NodesOrOther,Data}|Ts],Spec) when is_list(NodesOrOther) -> case lists:all(fun(Test) -> is_node(Test,Spec#testspec.nodes) end, NodesOrOther) of true -> - Ts1 = separate(NodesOrOther,Tag,[Data],Ts,Spec#testspec.nodes), + Ts1 = per_node(NodesOrOther,Tag,[Data],Ts,Spec#testspec.nodes), add_tests(Ts1,Spec); false -> add_tests([{Tag,all_nodes,{NodesOrOther,Data}}|Ts],Spec) @@ -1058,12 +1056,12 @@ update_recorded(Tag,Node,Spec) -> end. %% create one test term per node -separate(Nodes,Tag,Data,Tests,Refs) -> - Separated = separate(Nodes,Tag,Data,Refs), +per_node(Nodes,Tag,Data,Tests,Refs) -> + Separated = per_node(Nodes,Tag,Data,Refs), Separated ++ Tests. -separate([N|Ns],Tag,Data,Refs) -> - [list_to_tuple([Tag,ref2node(N,Refs)|Data])|separate(Ns,Tag,Data,Refs)]; -separate([],_,_,_) -> +per_node([N|Ns],Tag,Data,Refs) -> + [list_to_tuple([Tag,ref2node(N,Refs)|Data])|per_node(Ns,Tag,Data,Refs)]; +per_node([],_,_,_) -> []. %% read the value for FieldName in record Rec#testspec -- cgit v1.2.3 From f48550f96253588fcc643fcd5774a3c3ccb49b8b Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 22 Jan 2013 01:23:41 +0100 Subject: Add tests and correct errors OTP-9881 --- lib/common_test/src/ct_config.erl | 5 +- lib/common_test/src/ct_testspec.erl | 266 +++++++++++++++++++++++------------- 2 files changed, 178 insertions(+), 93 deletions(-) (limited to 'lib/common_test/src') diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index b1d709bc75..ac4ffbb236 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -266,7 +266,10 @@ read_config_files_int([{Callback, File}|Files], FunToSave) -> read_config_files_int([], _FunToSave) -> ok. -store_config(Config, Callback, File) -> +store_config(Config, Callback, File) when is_tuple(Config) -> + store_config([Config], Callback, File); + +store_config(Config, Callback, File) when is_list(Config) -> [ets:insert(?attr_table, #ct_conf{key=Key, value=Val, diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index e698f13b9a..abd82a3176 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -93,7 +93,7 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) -> %% run_per_node/2 takes the Run list as input and returns a list %% of {Node,RunPerNode,[]} tuples where the tests have been sorted %% on a per node basis. -run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) -> +run_per_node([{{Node,Dir},Test}|Ts],Result,MergeTests) -> {value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result), Run1 = case MergeTests of false -> @@ -190,7 +190,7 @@ prepare_suites(_Node,_Dir,[],Run,Skip) -> prepare_cases(Node,Dir,Suite,Cases) -> case get_skipped_cases(Node,Dir,Suite,Cases) of - SkipAll=[{{Node,Dir},{Suite,_Cmt}}] -> % all cases to be skipped + SkipAll=[{{Node,Dir},{Suite,_Cmt}}] -> % all cases to be skipped %% note: this adds an 'all' test even if only skip is specified {[{{Node,Dir},{Suite,all}}],SkipAll}; Skipped -> @@ -248,70 +248,125 @@ collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) -> NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes), %% [Spec1,Spec2,...] means create one testpec record per Spec file %% [[Spec1,Spec2,...]] means merge all specs into one testspec record - {JoinSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; + {Join,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; true -> {false,Specs} end, + Specs2 = [filename:absname(S) || S <- Specs1], TS0 = #testspec{nodes=NodeRefs}, - %% remove specs without tests - Filter = fun({_,#testspec{tests=[]}}) -> false; - (_) -> true - end, - try create_specs(Specs1,TS0,Relaxed,JoinSpecs,{[],TS0},[]) of + + try create_specs(Specs2,TS0,Relaxed,Join) of {{[],_},SeparateTestSpecs} -> - lists:filter(Filter,SeparateTestSpecs); + filter_and_convert(SeparateTestSpecs); {{_,#testspec{tests=[]}},SeparateTestSpecs} -> - lists:filter(Filter,SeparateTestSpecs); - {{JoinedSpecs,JoinedTestSpec},SeparateTestSpecs} -> - [{JoinedSpecs,JoinedTestSpec} | - lists:filter(Filter,SeparateTestSpecs)] + filter_and_convert(SeparateTestSpecs); + {Joined,SeparateTestSpecs} -> + [filter_and_convert(Joined) | + filter_and_convert(SeparateTestSpecs)] catch _:Error -> Error end. -create_specs([],_,_,_,Joined,Separate) -> - {Joined,Separate}; -create_specs([Spec|Ss],TestSpec,Relaxed,JoinSpecs, - Joined={JSpecs,_},Separate) -> +filter_and_convert(Joined) when is_tuple(Joined) -> + hd(filter_and_convert([Joined])); +filter_and_convert([{_,#testspec{tests=[]}}|TSs]) -> + filter_and_convert(TSs); +filter_and_convert([{[{SpecFile,MergeTests}|SMs],TestSpec}|TSs]) -> + #testspec{config = CfgFiles} = TestSpec, + TestSpec1 = TestSpec#testspec{config = delete_dups(CfgFiles), + merge_tests = MergeTests}, + %% set the merge_tests value for the testspec to the value + %% of the first test spec in the set + [{[SpecFile | [SF || {SF,_} <- SMs]], TestSpec1} | filter_and_convert(TSs)]; +filter_and_convert([]) -> + []. + +delete_dups(Elems) -> + delete_dups1(lists:reverse(Elems),[]). + +delete_dups1([E|Es],Keep) -> + case lists:member(E,Es) of + true -> + delete_dups1(Es,Keep); + false -> + delete_dups1(Es,[E|Keep]) + end; +delete_dups1([],Keep) -> + Keep. + +create_specs(Specs,TestSpec,Relaxed,Join) -> + SpecsTree = create_spec_tree(Specs,TestSpec,Join,[]), + create_specs(SpecsTree,TestSpec,Relaxed). + +create_spec_tree([Spec|Specs],TS,JoinWithNext,Known) -> SpecDir = filename:dirname(filename:absname(Spec)), - TestSpec1 = TestSpec#testspec{spec_dir=SpecDir}, - case file:consult(Spec) of - {ok,Terms} -> - Terms1 = replace_names(Terms), - {Specs2Join,SepSpecs} = get_included_specs(Terms1,TestSpec1), - TestSpec2 = create_spec(Terms1,TestSpec1,Relaxed), - case {JoinSpecs,Specs2Join,SepSpecs} of - {true,[],[]} -> - create_specs(Ss,TestSpec2,Relaxed,JoinSpecs, - {JSpecs++[get_absdir(Spec,TestSpec2)], - TestSpec2},Separate); - {false,[],[]} -> - create_specs(Ss,TestSpec,Relaxed,JoinSpecs,Joined, - Separate++[{[get_absdir(Spec,TestSpec2)], - TestSpec2}]); - _ -> - {{JSpecs1,JTS1},Separate1} = - create_specs(Specs2Join,TestSpec2,Relaxed,true, - {[get_absdir(Spec,TestSpec2)], - TestSpec2},[]), - {Joined2,Separate2} = - create_specs(SepSpecs,TestSpec,Relaxed,false, - {[],TestSpec1},[]), - NewJoined = {JSpecs++JSpecs1,JTS1}, - NewSeparate = Separate++Separate1++ - [Joined2 | Separate2], - NextTestSpec = if not JoinSpecs -> TestSpec; - true -> JTS1 - end, - create_specs(Ss,NextTestSpec,Relaxed,JoinSpecs, - NewJoined,NewSeparate) - end; - {error,Reason} -> - ReasonStr = - lists:flatten(io_lib:format("~s", - [file:format_error(Reason)])), - throw({error,{Spec,ReasonStr}}) - end. + TS1 = TS#testspec{spec_dir=SpecDir}, + SpecAbsName = get_absfile(Spec,TS1), + case lists:member(SpecAbsName,Known) of + true -> + throw({error,{cyclic_reference,SpecAbsName}}); + false -> + case file:consult(SpecAbsName) of + {ok,Terms} -> + Terms1 = replace_names(Terms), + {InclJoin,InclSep} = get_included_specs(Terms1,TS1), + {SpecAbsName,Terms1, + create_spec_tree(InclJoin,TS,true,[SpecAbsName|Known]), + create_spec_tree(InclSep,TS,false,[SpecAbsName|Known]), + JoinWithNext, + create_spec_tree(Specs,TS,JoinWithNext,Known)}; + {error,Reason} -> + ReasonStr = + lists:flatten(io_lib:format("~s", + [file:format_error(Reason)])), + throw({error,{SpecAbsName,ReasonStr}}) + end + end; +create_spec_tree([],_TS,_JoinWithNext,_Known) -> + []. + +create_specs({Spec,Terms,InclJoin,InclSep,JoinWithNext,NextSpec}, + TestSpec,Relaxed) -> + SpecDir = filename:dirname(filename:absname(Spec)), + TestSpec1 = create_spec(Terms,TestSpec#testspec{spec_dir=SpecDir},Relaxed), + + {{JoinSpecs1,JoinTS1},Separate1} = create_specs(InclJoin,TestSpec1,Relaxed), + {{JoinSpecs2,JoinTS2},Separate2} = + case JoinWithNext of + true -> + create_specs(NextSpec,JoinTS1,Relaxed); + false -> + {{[],JoinTS1},[]} + end, + {SepJoinSpecs,Separate3} = create_specs(InclSep,TestSpec,Relaxed), + {SepJoinSpecs1,Separate4} = + case JoinWithNext of + true -> + {{[],TestSpec},[]}; + false -> + create_specs(NextSpec,TestSpec,Relaxed) + end, + + SpecInfo = {Spec,TestSpec1#testspec.merge_tests}, + AllSeparate = + [TSData || TSData = {Ss,_TS} <- Separate3++Separate1++ + [SepJoinSpecs]++Separate2++ + Separate4++[SepJoinSpecs1], + Ss /= []], + + case {JoinWithNext,JoinSpecs1} of + {true,_} -> + {{[SpecInfo|(JoinSpecs1++JoinSpecs2)],JoinTS2}, + AllSeparate}; + {false,[]} -> + {{[],TestSpec}, + [{[SpecInfo],TestSpec1}|AllSeparate]}; + {false,_} -> + {{[SpecInfo|(JoinSpecs1++JoinSpecs2)],JoinTS2}, + AllSeparate} + end; +create_specs([],TestSpec,_Relaxed) -> + {{[],TestSpec},[]}. create_spec(Terms,TestSpec,Relaxed) -> TS = #testspec{tests=Tests, logdir=LogDirs} = @@ -345,7 +400,8 @@ collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) -> %% reverse nodes and aliases initially to get the order of them right %% in case this spec is being joined with a previous one TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As), - nodes = lists:reverse(Ns)}), + nodes = lists:reverse(Ns), + merge_tests = true}), TestSpec2 = get_all_nodes(Terms1,TestSpec1), {Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2), add_tests(Terms2,TestSpec3). @@ -483,9 +539,9 @@ get_included_specs(Terms,TestSpec) -> get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Sep) -> Specs = case SpecOrSpecs of [File|_] when is_list(File) -> - [get_absdir(Spec,TestSpec) || Spec <- SpecOrSpecs]; + [get_absfile(Spec,TestSpec) || Spec <- SpecOrSpecs]; [Ch|_] when is_integer(Ch) -> - [get_absdir(SpecOrSpecs,TestSpec)] + [get_absfile(SpecOrSpecs,TestSpec)] end, if How == join -> get_included_specs(Ts,TestSpec,Join++Specs,Sep); @@ -1118,14 +1174,21 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when {[Gr],Cases}; true -> {Gr,Cases} end || Gr <- Groups], - case lists:keysearch({Node,Dir},1,Tests) of - {value,{{Node,Dir},[{all,_}]}} -> - Tests; - {value,{{Node,Dir},Suites0}} -> - Suites1 = insert_groups1(Suite,Groups1,Suites0), - insert_in_order({{Node,Dir},Suites1},Tests); - false -> - insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests) + {Tests1,Done} = + lists:foldr(fun(All={{N,D},[{all,_}]},{Replaced,_}) when N == Node, + D == Dir -> + {[All|Replaced],true}; + ({{N,D},Suites0},{Replaced,_}) when N == Node, + D == Dir -> + Suites1 = insert_groups1(Suite,Groups1,Suites0), + {[{{N,D},Suites1}|Replaced],true}; + (T,{Replaced,Match}) -> + {[T|Replaced],Match} + end, {[],false}, Tests), + if not Done -> + Tests ++ [{{Node,Dir},[{Suite,Groups1}]}]; + true -> + Tests1 end; insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests) when is_atom(Case) -> @@ -1163,14 +1226,21 @@ insert_groups2([],GrAndCases) -> insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) -> append({{Node,Dir},[{Suite,Cases}]},Tests); insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) -> - case lists:keysearch({Node,Dir},1,Tests) of - {value,{{Node,Dir},[{all,_}]}} -> - Tests; - {value,{{Node,Dir},Suites0}} -> - Suites1 = insert_cases1(Suite,Cases,Suites0), - insert_in_order({{Node,Dir},Suites1},Tests); - false -> - insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests) + {Tests1,Done} = + lists:foldr(fun(All={{N,D},[{all,_}]},{Replaced,_}) when N == Node, + D == Dir -> + {[All|Replaced],true}; + ({{N,D},Suites0},{Replaced,_}) when N == Node, + D == Dir -> + Suites1 = insert_cases1(Suite,Cases,Suites0), + {[{{N,D},Suites1}|Replaced],true}; + (T,{Replaced,Match}) -> + {[T|Replaced],Match} + end, {[],false}, Tests), + if not Done -> + Tests ++ [{{Node,Dir},[{Suite,Cases}]}]; + true -> + Tests1 end; insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) -> insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests). @@ -1211,15 +1281,23 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when append({{Node,Dir},Suites1},Tests); skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> - Suites = - case lists:keysearch({Node,Dir},1,Tests) of - {value,{{Node,Dir},Suites0}} -> - Suites0; - false -> - [] - end, - Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites), - insert_in_order({{Node,Dir},Suites1},Tests); + {Tests1,Done} = + lists:foldr(fun({{N,D},Suites0},{Replaced,_}) when N == Node, + D == Dir -> + Suites1 = skip_groups1(Suite, + [{Gr,Cases} || Gr <- Groups], + Cmt,Suites0), + {[{{N,D},Suites1}|Replaced],true}; + (T,{Replaced,Match}) -> + {[T|Replaced],Match} + end, {[],false}, Tests), + if not Done -> + Tests ++ [{{Node,Dir},skip_groups1(Suite, + [{Gr,Cases} || Gr <- Groups], + Cmt,[])}]; + true -> + Tests1 + end; skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) when is_atom(Case) -> Cases = if Case == all -> all; true -> [Case] end, @@ -1241,15 +1319,19 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) -> Suites1 = skip_cases1(Suite,Cases,Cmt,[]), append({{Node,Dir},Suites1},Tests); skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) -> - Suites = - case lists:keysearch({Node,Dir},1,Tests) of - {value,{{Node,Dir},Suites0}} -> - Suites0; - false -> - [] - end, - Suites1 = skip_cases1(Suite,Cases,Cmt,Suites), - insert_in_order({{Node,Dir},Suites1},Tests); + {Tests1,Done} = + lists:foldr(fun({{N,D},Suites0},{Replaced,_}) when N == Node, + D == Dir -> + Suites1 = skip_cases1(Suite,Cases,Cmt,Suites0), + {[{{N,D},Suites1}|Replaced],true}; + (T,{Replaced,Match}) -> + {[T|Replaced],Match} + end, {[],false}, Tests), + if not Done -> + Tests ++ [{{Node,Dir},skip_cases1(Suite,Cases,Cmt,[])}]; + true -> + Tests1 + end; skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) -> skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests). -- cgit v1.2.3 From 9230b38f318d916258168fafb2878f225fb3052b Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 24 Jan 2013 12:20:02 +0100 Subject: Update documentation OTP-9881 --- lib/common_test/src/ct.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/common_test/src') diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 8eafdff29f..853a1582cf 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -144,8 +144,8 @@ run(TestDirs) -> %%% @spec run_test(Opts) -> Result %%% Opts = [OptTuples] %%% OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} | -%%% {testcase,Cases} | {spec,TestSpecs} | {label,Label} | -%%% {config,CfgFiles} | {userconfig, UserConfig} | +%%% {testcase,Cases} | {spec,TestSpecs} | {join_specs,Bool} | +%%% {label,Label} | {config,CfgFiles} | {userconfig, UserConfig} | %%% {allow_user_terms,Bool} | {logdir,LogDir} | %%% {silent_connections,Conns} | {stylesheet,CSSFile} | %%% {cover,CoverSpecFile} | {cover_stop,Bool} | {step,StepOpts} | -- cgit v1.2.3