diff options
author | Peter Andersson <peppe@erlang.org> | 2013-01-14 15:12:01 +0100 |
---|---|---|
committer | Peter Andersson <peppe@erlang.org> | 2013-01-24 10:53:58 +0100 |
commit | 102c8167c6960233102aa6f2fabec5416882b74f (patch) | |
tree | 4b6d7d7beba7c3388215bd792d9ca0f5481e36cf /lib | |
parent | adc50b681c3d3bd6d98f6c3f88dad4a306d3a239 (diff) | |
download | otp-102c8167c6960233102aa6f2fabec5416882b74f.tar.gz otp-102c8167c6960233102aa6f2fabec5416882b74f.tar.bz2 otp-102c8167c6960233102aa6f2fabec5416882b74f.zip |
Implement support for including test specifications
OTP-9881
Diffstat (limited to 'lib')
-rw-r--r-- | lib/common_test/src/ct_run.erl | 423 | ||||
-rw-r--r-- | lib/common_test/src/ct_testspec.erl | 134 | ||||
-rw-r--r-- | lib/common_test/test/ct_testspec_2_SUITE.erl | 18 |
3 files changed, 303 insertions, 272 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 50723a763c..742e287493 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -330,33 +330,33 @@ script_start1(Parent, Args) -> true end, - StartOpts = #opts{label = Label, profile = Profile, - vts = Vts, shell = Shell, - cover = Cover, cover_stop = CoverStop, - logdir = LogDir, logopts = LogOpts, - basic_html = BasicHtml, - verbosity = Verbosity, - event_handlers = EvHandlers, - ct_hooks = CTHooks, - enable_builtin_hooks = EnableBuiltinHooks, - auto_compile = AutoCompile, - include = IncludeDirs, - silent_connections = SilentConns, - stylesheet = Stylesheet, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir, - starter = script}, - + Opts = #opts{label = Label, profile = Profile, + vts = Vts, shell = Shell, + cover = Cover, cover_stop = CoverStop, + logdir = LogDir, logopts = LogOpts, + basic_html = BasicHtml, + verbosity = Verbosity, + event_handlers = EvHandlers, + ct_hooks = CTHooks, + enable_builtin_hooks = EnableBuiltinHooks, + auto_compile = AutoCompile, + include = IncludeDirs, + silent_connections = SilentConns, + stylesheet = Stylesheet, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir, + starter = script}, + %% check if log files should be refreshed or go on to run tests... - Result = run_or_refresh(StartOpts, Args), + Result = run_or_refresh(Opts, Args), %% send final results to starting process waiting in script_start/0 Parent ! {self(), Result}. -run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> +run_or_refresh(Opts = #opts{logdir = LogDir}, Args) -> case proplists:get_value(refresh_logs, Args) of undefined -> - script_start2(StartOpts, Args); + script_start2(Opts, Args); Refresh -> LogDir1 = case Refresh of [] -> which(logdir,LogDir); @@ -386,8 +386,8 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) -> end end. -script_start2(StartOpts = #opts{vts = undefined, - shell = undefined}, Args) -> +script_start2(Opts = #opts{vts = undefined, + shell = undefined}, Args) -> case proplists:get_value(spec, Args) of Specs when Specs =/= [], Specs =/= undefined -> Specs1 = get_start_opt(join_specs, [Specs], Specs, Args), @@ -397,38 +397,38 @@ script_start2(StartOpts = #opts{vts = undefined, {E,Reason} when E == error ; E == 'EXIT' -> {error,Reason}; TestSpecData -> - execute_testspecs(TestSpecData, StartOpts, Args, []) + execute_all_specs(TestSpecData, Opts, Args, []) end; [] -> {error,no_testspec_specified}; _ -> % no testspec used %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), - TheLogDir = which(logdir, StartOpts#opts.logdir), + TheLogDir = which(logdir, Opts#opts.logdir), case check_and_install_configfiles(InitConfig, TheLogDir, - StartOpts) of + Opts) of ok -> % go on read tests from start flags - script_start3(StartOpts#opts{config=InitConfig, - logdir=TheLogDir}, Args); + script_start3(Opts#opts{config=InitConfig, + logdir=TheLogDir}, Args); Error -> Error end end; -script_start2(StartOpts, Args) -> +script_start2(Opts, Args) -> %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), - LogDir = which(logdir, StartOpts#opts.logdir), - case check_and_install_configfiles(InitConfig, LogDir, StartOpts) of + LogDir = which(logdir, Opts#opts.logdir), + case check_and_install_configfiles(InitConfig, LogDir, Opts) of ok -> % go on read tests from start flags - script_start3(StartOpts#opts{config=InitConfig, - logdir=LogDir}, Args); + script_start3(Opts#opts{config=InitConfig, + logdir=LogDir}, Args); Error -> Error end. -execute_testspecs([], _, _, Result) -> +execute_all_specs([], _, _, Result) -> Result1 = lists:reverse(Result), case lists:keysearch('EXIT', 1, Result1) of {value,{_,_,ExitReason}} -> @@ -447,67 +447,93 @@ execute_testspecs([], _, _, Result) -> end end; -execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> - SpecStartOpts = get_data_for_node(TS, node()), - - Label = choose_val(StartOpts#opts.label, - SpecStartOpts#opts.label), - - Profile = choose_val(StartOpts#opts.profile, - SpecStartOpts#opts.profile), - - LogDir = choose_val(StartOpts#opts.logdir, - SpecStartOpts#opts.logdir), - - AllLogOpts = merge_vals([StartOpts#opts.logopts, - SpecStartOpts#opts.logopts]), +execute_all_specs([{Specs,TS} | TSs], Opts, Args, Result) -> + CombinedOpts = combine_test_opts(TS, Specs, Opts), + try execute_one_spec(TS, CombinedOpts, Args) of + ExecResult -> + execute_all_specs(TSs, Opts, Args, [ExecResult|Result]) + catch + _ : ExitReason -> + execute_all_specs(TSs, Opts, Args, + [{'EXIT',self(),ExitReason}|Result]) + end. + +execute_one_spec(TS, Opts, Args) -> + %% read config/userconfig from start flags + InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), + %% merge config from start flags with config from testspec + AllConfig = merge_vals([InitConfig, Opts#opts.config]), + case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of + ok -> % read tests from spec + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir}, Args); + Error -> + Error + end. + +combine_test_opts(TS, Specs, Opts) -> + TSOpts = get_data_for_node(TS, node()), + + Label = choose_val(Opts#opts.label, + TSOpts#opts.label), + + Profile = choose_val(Opts#opts.profile, + TSOpts#opts.profile), + + LogDir = choose_val(Opts#opts.logdir, + TSOpts#opts.logdir), + + AllLogOpts = merge_vals([Opts#opts.logopts, + TSOpts#opts.logopts]), AllVerbosity = - merge_keyvals([StartOpts#opts.verbosity, - SpecStartOpts#opts.verbosity]), + merge_keyvals([Opts#opts.verbosity, + TSOpts#opts.verbosity]), AllSilentConns = - merge_vals([StartOpts#opts.silent_connections, - SpecStartOpts#opts.silent_connections]), + merge_vals([Opts#opts.silent_connections, + TSOpts#opts.silent_connections]), Cover = - choose_val(StartOpts#opts.cover, - SpecStartOpts#opts.cover), + choose_val(Opts#opts.cover, + TSOpts#opts.cover), CoverStop = - choose_val(StartOpts#opts.cover_stop, - SpecStartOpts#opts.cover_stop), + choose_val(Opts#opts.cover_stop, + TSOpts#opts.cover_stop), MultTT = - choose_val(StartOpts#opts.multiply_timetraps, - SpecStartOpts#opts.multiply_timetraps), + choose_val(Opts#opts.multiply_timetraps, + TSOpts#opts.multiply_timetraps), ScaleTT = - choose_val(StartOpts#opts.scale_timetraps, - SpecStartOpts#opts.scale_timetraps), - + choose_val(Opts#opts.scale_timetraps, + TSOpts#opts.scale_timetraps), + CreatePrivDir = - choose_val(StartOpts#opts.create_priv_dir, - SpecStartOpts#opts.create_priv_dir), - + choose_val(Opts#opts.create_priv_dir, + TSOpts#opts.create_priv_dir), + AllEvHs = - merge_vals([StartOpts#opts.event_handlers, - SpecStartOpts#opts.event_handlers]), - + merge_vals([Opts#opts.event_handlers, + TSOpts#opts.event_handlers]), + AllCTHooks = merge_vals( - [StartOpts#opts.ct_hooks, - SpecStartOpts#opts.ct_hooks]), - + [Opts#opts.ct_hooks, + TSOpts#opts.ct_hooks]), + EnableBuiltinHooks = choose_val( - StartOpts#opts.enable_builtin_hooks, - SpecStartOpts#opts.enable_builtin_hooks), - + Opts#opts.enable_builtin_hooks, + TSOpts#opts.enable_builtin_hooks), + Stylesheet = - choose_val(StartOpts#opts.stylesheet, - SpecStartOpts#opts.stylesheet), - - AllInclude = merge_vals([StartOpts#opts.include, - SpecStartOpts#opts.include]), + choose_val(Opts#opts.stylesheet, + TSOpts#opts.stylesheet), + + AllInclude = merge_vals([Opts#opts.include, + TSOpts#opts.include]), application:set_env(common_test, include, AllInclude), - + AutoCompile = - case choose_val(StartOpts#opts.auto_compile, - SpecStartOpts#opts.auto_compile) of + case choose_val(Opts#opts.auto_compile, + TSOpts#opts.auto_compile) of undefined -> true; ACBool -> @@ -518,8 +544,8 @@ execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> end, BasicHtml = - case choose_val(StartOpts#opts.basic_html, - SpecStartOpts#opts.basic_html) of + case choose_val(Opts#opts.basic_html, + TSOpts#opts.basic_html) of undefined -> false; BHBool -> @@ -527,52 +553,27 @@ execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) -> BHBool), BHBool end, - - Opts = StartOpts#opts{label = Label, - profile = Profile, - testspecs = Specs, - cover = Cover, - cover_stop = CoverStop, - logdir = LogDir, - logopts = AllLogOpts, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = SpecStartOpts#opts.config, - event_handlers = AllEvHs, - ct_hooks = AllCTHooks, - enable_builtin_hooks = - EnableBuiltinHooks, - stylesheet = Stylesheet, - auto_compile = AutoCompile, - include = AllInclude, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir}, - - try execute_testspec(TS, Opts, Args) of - ExecResult -> - execute_testspecs(TSs, StartOpts, Args, [ExecResult | Result]) - catch - _ : ExitReason -> - execute_testspecs(TSs, StartOpts, Args, - [{'EXIT',self(),ExitReason} | Result]) - end. - -execute_testspec(TS, Opts, Args) -> - %% read config/userconfig from start flags - InitConfig = ct_config:prepare_config_list(Args), - TheLogDir = which(logdir, Opts#opts.logdir), - %% merge config from start flags with config from testspec - AllConfig = merge_vals([InitConfig, Opts#opts.config]), - case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of - ok -> % read tests from spec - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig, - logdir=TheLogDir}, Args); - Error -> - Error - end. + + Opts#opts{label = Label, + profile = Profile, + testspecs = Specs, + cover = Cover, + cover_stop = CoverStop, + logdir = which(logdir, LogDir), + logopts = AllLogOpts, + basic_html = BasicHtml, + verbosity = AllVerbosity, + silent_connections = AllSilentConns, + config = TSOpts#opts.config, + event_handlers = AllEvHs, + ct_hooks = AllCTHooks, + enable_builtin_hooks = EnableBuiltinHooks, + stylesheet = Stylesheet, + auto_compile = AutoCompile, + include = AllInclude, + multiply_timetraps = MultTT, + scale_timetraps = ScaleTT, + create_priv_dir = CreatePrivDir}. check_and_install_configfiles( Configs, LogDir, #opts{ @@ -593,12 +594,12 @@ check_and_install_configfiles( {error,{cant_load_callback_module,Info}} end. -script_start3(StartOpts, Args) -> - StartOpts1 = get_start_opt(step, - fun(Step) -> - StartOpts#opts{step = Step, - cover = undefined} - end, StartOpts, Args), +script_start3(Opts, Args) -> + Opts1 = get_start_opt(step, + fun(Step) -> + Opts#opts{step = Step, + cover = undefined} + end, Opts, Args), case {proplists:get_value(dir, Args), proplists:get_value(suite, Args), groups_and_cases(proplists:get_value(group, Args), @@ -612,17 +613,17 @@ script_start3(StartOpts, Args) -> {error,no_dir_specified}; {Dirs,undefined,[]} when is_list(Dirs) -> - script_start4(StartOpts#opts{tests = tests(Dirs)}, Args); + script_start4(Opts#opts{tests = tests(Dirs)}, Args); {undefined,Suites,[]} when is_list(Suites) -> Ts = tests([suite_to_test(S) || S <- Suites]), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); {undefined,Suite,GsAndCs} when is_list(Suite) -> case [suite_to_test(S) || S <- Suite] of DirMods = [_] -> Ts = tests(DirMods, GsAndCs), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); [_,_|_] -> {error,multiple_suites_and_cases}; _ -> @@ -636,10 +637,10 @@ script_start3(StartOpts, Args) -> case [suite_to_test(Dir,S) || S <- Suite] of DirMods when GsAndCs == [] -> Ts = tests(DirMods), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); DirMods = [_] when GsAndCs /= [] -> Ts = tests(DirMods, GsAndCs), - script_start4(StartOpts1#opts{tests = Ts}, Args); + script_start4(Opts1#opts{tests = Ts}, Args); [_,_|_] when GsAndCs /= [] -> {error,multiple_suites_and_cases}; _ -> @@ -650,8 +651,8 @@ script_start3(StartOpts, Args) -> {error,incorrect_start_options}; {undefined,undefined,_} -> - if StartOpts#opts.vts ; StartOpts#opts.shell -> - script_start4(StartOpts#opts{tests = []}, Args); + if Opts#opts.vts ; Opts#opts.shell -> + script_start4(Opts#opts{tests = []}, Args); true -> script_usage(), {error,missing_start_options} @@ -781,6 +782,7 @@ script_usage() -> "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]" "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]" "\n\t[-allow_user_terms]" + "\n\t[-join_specs]" "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" @@ -1075,105 +1077,60 @@ run_test2(StartOpts) -> end. run_spec_file(Relaxed, - Opts = #opts{testspecs = Specs, config = CfgFiles}, + Opts = #opts{testspecs = Specs}, StartOpts) -> Specs1 = case Specs of [X|_] when is_integer(X) -> [Specs]; _ -> Specs end, AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1), - log_ts_names(AbsSpecs), - case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of + AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts), + case catch ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of {Error,CTReason} when Error == error ; Error == 'EXIT' -> exit({error,CTReason}); - TS -> - SpecOpts = get_data_for_node(TS, node()), - Label = choose_val(Opts#opts.label, - SpecOpts#opts.label), - Profile = choose_val(Opts#opts.profile, - SpecOpts#opts.profile), - LogDir = choose_val(Opts#opts.logdir, - SpecOpts#opts.logdir), - AllLogOpts = merge_vals([Opts#opts.logopts, - SpecOpts#opts.logopts]), - Stylesheet = choose_val(Opts#opts.stylesheet, - SpecOpts#opts.stylesheet), - AllVerbosity = merge_keyvals([Opts#opts.verbosity, - SpecOpts#opts.verbosity]), - AllSilentConns = merge_vals([Opts#opts.silent_connections, - SpecOpts#opts.silent_connections]), - AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), - Cover = choose_val(Opts#opts.cover, - SpecOpts#opts.cover), - CoverStop = choose_val(Opts#opts.cover_stop, - SpecOpts#opts.cover_stop), - MultTT = choose_val(Opts#opts.multiply_timetraps, - SpecOpts#opts.multiply_timetraps), - ScaleTT = choose_val(Opts#opts.scale_timetraps, - SpecOpts#opts.scale_timetraps), - CreatePrivDir = choose_val(Opts#opts.create_priv_dir, - SpecOpts#opts.create_priv_dir), - AllEvHs = merge_vals([Opts#opts.event_handlers, - SpecOpts#opts.event_handlers]), - AllInclude = merge_vals([Opts#opts.include, - SpecOpts#opts.include]), - AllCTHooks = merge_vals([Opts#opts.ct_hooks, - SpecOpts#opts.ct_hooks]), - EnableBuiltinHooks = choose_val(Opts#opts.enable_builtin_hooks, - SpecOpts#opts.enable_builtin_hooks), - - application:set_env(common_test, include, AllInclude), - - AutoCompile = case choose_val(Opts#opts.auto_compile, - SpecOpts#opts.auto_compile) of - undefined -> - true; - ACBool -> - application:set_env(common_test, auto_compile, - ACBool), - ACBool - end, + TestSpecData -> + run_all_specs(TestSpecData, Opts, StartOpts, []) + end. - BasicHtml = case choose_val(Opts#opts.basic_html, - SpecOpts#opts.basic_html) of - undefined -> - false; - BHBool -> - application:set_env(common_test, basic_html, - BHBool), - BHBool - end, - - Opts1 = Opts#opts{label = Label, - profile = Profile, - cover = Cover, - cover_stop = CoverStop, - logdir = which(logdir, LogDir), - logopts = AllLogOpts, - stylesheet = Stylesheet, - basic_html = BasicHtml, - verbosity = AllVerbosity, - silent_connections = AllSilentConns, - config = AllConfig, - event_handlers = AllEvHs, - auto_compile = AutoCompile, - include = AllInclude, - testspecs = AbsSpecs, - multiply_timetraps = MultTT, - scale_timetraps = ScaleTT, - create_priv_dir = CreatePrivDir, - ct_hooks = AllCTHooks, - enable_builtin_hooks = EnableBuiltinHooks - }, - - case check_and_install_configfiles(AllConfig,Opts1#opts.logdir, - Opts1) of - ok -> - {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); - {error,GCFReason} -> - exit({error,GCFReason}) +run_all_specs([], _, _, TotResult) -> + TotResult1 = lists:reverse(TotResult), + case lists:keysearch('EXIT', 1, TotResult1) of + {value,{_,_,ExitReason}} -> + exit(ExitReason); + false -> + case lists:keysearch(error, 1, TotResult1) of + {value,Error} -> + Error; + false -> + lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}}, + {Ok1,Fail1,{UserSkip1,AutoSkip1}}) -> + {Ok1+Ok,Fail1+Fail, + {UserSkip1+UserSkip, + AutoSkip1+AutoSkip}} + end, {0,0,{0,0}}, TotResult1) end + end; + +run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) -> + log_ts_names(Specs), + Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts), + AllConfig = merge_vals([Opts#opts.config, TSConfig]), + try run_one_spec(TS, Combined#opts{config = AllConfig}, StartOpts) of + Result -> + run_all_specs(TSs, Opts, StartOpts, [Result | TotResult]) + catch + _ : Reason -> + run_all_specs(TSs, Opts, StartOpts, [{error,Reason} | TotResult]) + end. + +run_one_spec(TS, CombinedOpts, StartOpts) -> + #opts{logdir = Logdir, config = Config} = CombinedOpts, + case check_and_install_configfiles(Config, Logdir, CombinedOpts) of + ok -> + {Run,Skip} = ct_testspec:prepare_tests(TS, node()), + reformat_result(catch do_run(Run, Skip, CombinedOpts, StartOpts)); + Error -> + Error end. run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, @@ -2927,6 +2884,10 @@ opts2args(EnvStartOpts) -> [{allow_user_terms,[]}]; ({allow_user_terms,false}) -> []; + ({join_specs,true}) -> + [{join_specs,[]}]; + ({join_specs,false}) -> + []; ({auto_compile,false}) -> [{no_auto_compile,[]}]; ({auto_compile,true}) -> diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 9a7a384ebb..3d9351b71b 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -241,45 +241,71 @@ get_skipped_cases1(_,_,_,[]) -> %%% collect_tests_from_file reads a testspec file and returns a record %%% containing the data found. -collect_tests_from_file(Specs, Relaxed) -> +collect_tests_from_file(Specs,Relaxed) -> collect_tests_from_file(Specs,[node()],Relaxed). collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) -> NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes), %% [Spec1,Spec2,...] means create one testpec record per Spec file %% [[Spec1,Spec2,...]] means merge all specs into one testspec record - {MergeSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; + {JoinSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)}; true -> {false,Specs} end, - catch create_specs(Specs1,Specs1,#testspec{nodes=NodeRefs}, - Relaxed,MergeSpecs,[]). + TS0 = #testspec{nodes=NodeRefs}, + %% remove specs without tests + Filter = fun({_,#testspec{tests=[]}}) -> false; + (_) -> true + end, + try create_specs(Specs1,TS0,Relaxed,JoinSpecs,{[],TS0},[]) of + {{[],_},AdditionalTestSpecs} -> + lists:filter(Filter,AdditionalTestSpecs); + {{_,#testspec{tests=[]}},AdditionalTestSpecs} -> + lists:filter(Filter,AdditionalTestSpecs); + {{JoinedSpecs,JoinedTestSpec},AdditionalTestSpecs} -> + [{JoinedSpecs,JoinedTestSpec} | + lists:filter(Filter,AdditionalTestSpecs)] + catch + _:Error -> + Error + end. -create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) -> +create_specs([],_,_,_,Joined,Additional) -> + {Joined,Additional}; +create_specs([Spec|Ss],TestSpec,Relaxed,JoinSpecs, + Joined={JSpecs,_},Additional) -> SpecDir = filename:dirname(filename:absname(Spec)), + TestSpec1 = TestSpec#testspec{spec_dir=SpecDir}, case file:consult(Spec) of - {ok,Terms} -> - case collect_tests(Terms, - TestSpec#testspec{spec_dir=SpecDir}, - Relaxed) of - TS = #testspec{tests=Tests, logdir=LogDirs} when - Ss == [], MergeSpecs == true -> - LogDirs1 = lists:delete(".",LogDirs) ++ ["."], - [{Specs,TS#testspec{tests=lists:flatten(Tests), - logdir=LogDirs1}}]; - TS = #testspec{tests=Tests, logdir=LogDirs} when - Ss == [], MergeSpecs == false -> - LogDirs1 = lists:delete(".",LogDirs) ++ ["."], - TSRet = {[Spec],TS#testspec{tests=lists:flatten(Tests), - logdir=LogDirs1}}, - lists:reverse([TSRet|Saved]); - TS = #testspec{alias = As, nodes = Ns} when - MergeSpecs == true -> - TS1 = TS#testspec{alias = lists:reverse(As), - nodes = lists:reverse(Ns)}, - create_specs(Ss,Specs,TS1,Relaxed,MergeSpecs,[]); - TS when MergeSpecs == false -> - create_specs(Ss,Specs,TestSpec,Relaxed,MergeSpecs, - [{[Spec],TS}|Saved]) + {ok,Terms} -> + Terms1 = replace_names(Terms), + {Specs2Join,Specs2Add} = get_included_specs(Terms1,TestSpec1), + TestSpec2 = create_spec(Terms1,TestSpec1, + Relaxed,JoinSpecs), + case {JoinSpecs,Specs2Join,Specs2Add} of + {true,[],[]} -> + create_specs(Ss,TestSpec2,Relaxed,JoinSpecs, + {JSpecs++[get_absdir(Spec,TestSpec2)], + TestSpec2},Additional); + {false,[],[]} -> + create_specs(Ss,TestSpec,Relaxed,JoinSpecs,Joined, + Additional++[{[get_absdir(Spec,TestSpec2)], + TestSpec2}]); + _ -> + {{JSpecs1,JTS1},Additional1} = + create_specs(Specs2Join,TestSpec2,Relaxed,true, + {[get_absdir(Spec,TestSpec2)], + TestSpec2},[]), + {Joined2,Additional2} = + create_specs(Specs2Add,TestSpec,Relaxed,false, + {[],TestSpec1},[]), + NewJoined = {JSpecs++JSpecs1,JTS1}, + NewAdditional = Additional++Additional1++ + [Joined2 | Additional2], + NextTestSpec = if not JoinSpecs -> TestSpec; + true -> JTS1 + end, + create_specs(Ss,NextTestSpec,Relaxed,JoinSpecs, + NewJoined,NewAdditional) end; {error,Reason} -> ReasonStr = @@ -287,6 +313,14 @@ create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) -> [file:format_error(Reason)])), throw({error,{Spec,ReasonStr}}) end. + +create_spec(Terms,TestSpec,Relaxed,JoinSpecs) -> + TS = #testspec{tests=Tests, logdir=LogDirs} = + collect_tests({false,Terms},TestSpec,Relaxed), + + LogDirs1 = lists:delete(".",LogDirs) ++ ["."], + TS#testspec{tests=lists:flatten(Tests), + logdir=LogDirs1}. collect_tests_from_list(Terms,Relaxed) -> collect_tests_from_list(Terms,[node()],Relaxed). @@ -294,8 +328,8 @@ collect_tests_from_list(Terms,Relaxed) -> collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) -> {ok,Cwd} = file:get_cwd(), NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes), - case catch collect_tests(Terms,#testspec{nodes=NodeRefs, - spec_dir=Cwd}, + case catch collect_tests({true,Terms},#testspec{nodes=NodeRefs, + spec_dir=Cwd}, Relaxed) of E = {error,_} -> E; @@ -305,10 +339,15 @@ collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) -> TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1} end. -collect_tests(Terms,TestSpec,Relaxed) -> +collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) -> put(relaxed,Relaxed), - Terms1 = replace_names(Terms), - TestSpec1 = get_global(Terms1,TestSpec), + Terms1 = if Replace -> replace_names(Terms); + true -> Terms + end, + %% reverse nodes and aliases initially to get the order of them right + %% in case this spec is being joined with a previous one + TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As), + nodes = lists:reverse(Ns)}), TestSpec2 = get_all_nodes(Terms1,TestSpec1), {Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2), add_tests(Terms2,TestSpec3). @@ -438,9 +477,30 @@ replace_names_in_node1(NodeStr,Defs=[{Name,Replacement}|Ds]) -> replace_names_in_node1(NodeStr,[]) -> NodeStr. +%% look for other specification files, either to join with the +%% current spec, or execute as additional test runs +get_included_specs(Terms,TestSpec) -> + get_included_specs(Terms,TestSpec,[],[]). + +get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Add) -> + Specs = case SpecOrSpecs of + [File|_] when is_list(File) -> + [get_absdir(Spec,TestSpec) || Spec <- SpecOrSpecs]; + [Ch|_] when is_integer(Ch) -> + [get_absdir(SpecOrSpecs,TestSpec)] + end, + if How == join -> + get_included_specs(Ts,TestSpec,Join++Specs,Add); + true -> + get_included_specs(Ts,TestSpec,Join,Add++Specs) + end; +get_included_specs([_|Ts],TestSpec,Join,Add) -> + get_included_specs(Ts,TestSpec,Join,Add); +get_included_specs([],_,Join,Add) -> + {Join,Add}. %% global terms that will be used for analysing all other terms in the spec -get_global([{merge_tests,Bool} | Ts], Spec) -> +get_global([{merge_tests,Bool}|Ts],Spec) -> get_global(Ts,Spec#testspec{merge_tests=Bool}); %% the 'define' term replaces the 'alias' and 'node' terms, but we need to keep @@ -810,7 +870,10 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled add_tests([{node,_,_}|Ts],Spec) -> % handled add_tests(Ts,Spec); -add_tests([{merge_tests, _} | Ts], Spec) -> % handled +add_tests([{merge_tests,_} | Ts], Spec) -> % handled + add_tests(Ts,Spec); + +add_tests([{specs,_,_} | Ts], Spec) -> % handled add_tests(Ts,Spec); %% -------------------------------------------------- @@ -1279,6 +1342,7 @@ is_node([],_) -> valid_terms() -> [ {define,3}, + {specs,3}, {node,3}, {cover,2}, {cover,3}, diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl index 9d2dc84ad3..c150686482 100644 --- a/lib/common_test/test/ct_testspec_2_SUITE.erl +++ b/lib/common_test/test/ct_testspec_2_SUITE.erl @@ -479,7 +479,7 @@ multiple_specs(_Config) -> "multiple_specs.1.spec"), SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir, "multiple_specs.2.spec"), - FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2], + FileResult = ct_testspec:collect_tests_from_file([[SpecFile1,SpecFile2]], false), ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]), @@ -524,7 +524,7 @@ multiple_specs(_Config) -> %%% misc_config_terms(_Config) -> CfgDir = "../cfgs/to1", - + TODir = "../tests/to1", Spec = [{node,x,n1@h1},{node,y,n2@h2}, @@ -554,7 +554,9 @@ misc_config_terms(_Config) -> {create_priv_dir,[auto_per_tc]}, {create_priv_dir,n1@h1,[manual_per_tc]}, - {create_priv_dir,n2@h2,[auto_per_run]} + {create_priv_dir,n2@h2,[auto_per_run]}, + + {suites,n1@h1,TODir,[x_SUITE]} ], {ok,SpecDir} = file:get_cwd(), @@ -599,7 +601,9 @@ misc_config_terms(_Config) -> {n2@h2,CSS2}], create_priv_dir = [{Node,[auto_per_tc]}, {n1@h1,[manual_per_tc]}, - {n2@h2,[auto_per_run]}] + {n2@h2,[auto_per_run]}], + tests = [{{n1@h1,get_absdir(filename:join(SpecDir,TODir))}, + [{x_SUITE,[all]}]}] }, verify_result(Verify,ListResult,FileResult). @@ -688,10 +692,10 @@ define_names_1(_Config) -> %%% HELP FUNCTIONS %%%----------------------------------------------------------------- -verify_result(Verify,ListResult,FileResult) -> +verify_result(VerificationRec,ListResult,FileResult) -> {_,TSLTuples} = rec2proplist(ListResult), {_,TSFTuples} = rec2proplist(FileResult), - {_,VTuples} = rec2proplist(Verify), + {_,VTuples} = rec2proplist(VerificationRec), VResult = (catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) -> {TSL,TSF}; @@ -720,6 +724,8 @@ read_config(S) -> rec2proplist(E={error,_What}) -> exit({invalid_testspec_record,E}); +rec2proplist([{Specs,Rec}]) when is_list(Specs) -> + rec2proplist(Rec); rec2proplist(Rec) -> [RecName|RecList] = tuple_to_list(Rec), FieldNames = |