aboutsummaryrefslogtreecommitdiffstats
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/common_test/src/ct_run.erl423
-rw-r--r--lib/common_test/src/ct_testspec.erl134
-rw-r--r--lib/common_test/test/ct_testspec_2_SUITE.erl18
3 files changed, 303 insertions, 272 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 50723a763c..742e287493 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -330,33 +330,33 @@ script_start1(Parent, Args) ->
true
end,
- StartOpts = #opts{label = Label, profile = Profile,
- vts = Vts, shell = Shell,
- cover = Cover, cover_stop = CoverStop,
- logdir = LogDir, logopts = LogOpts,
- basic_html = BasicHtml,
- verbosity = Verbosity,
- event_handlers = EvHandlers,
- ct_hooks = CTHooks,
- enable_builtin_hooks = EnableBuiltinHooks,
- auto_compile = AutoCompile,
- include = IncludeDirs,
- silent_connections = SilentConns,
- stylesheet = Stylesheet,
- multiply_timetraps = MultTT,
- scale_timetraps = ScaleTT,
- create_priv_dir = CreatePrivDir,
- starter = script},
-
+ Opts = #opts{label = Label, profile = Profile,
+ vts = Vts, shell = Shell,
+ cover = Cover, cover_stop = CoverStop,
+ logdir = LogDir, logopts = LogOpts,
+ basic_html = BasicHtml,
+ verbosity = Verbosity,
+ event_handlers = EvHandlers,
+ ct_hooks = CTHooks,
+ enable_builtin_hooks = EnableBuiltinHooks,
+ auto_compile = AutoCompile,
+ include = IncludeDirs,
+ silent_connections = SilentConns,
+ stylesheet = Stylesheet,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT,
+ create_priv_dir = CreatePrivDir,
+ starter = script},
+
%% check if log files should be refreshed or go on to run tests...
- Result = run_or_refresh(StartOpts, Args),
+ Result = run_or_refresh(Opts, Args),
%% send final results to starting process waiting in script_start/0
Parent ! {self(), Result}.
-run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
+run_or_refresh(Opts = #opts{logdir = LogDir}, Args) ->
case proplists:get_value(refresh_logs, Args) of
undefined ->
- script_start2(StartOpts, Args);
+ script_start2(Opts, Args);
Refresh ->
LogDir1 = case Refresh of
[] -> which(logdir,LogDir);
@@ -386,8 +386,8 @@ run_or_refresh(StartOpts = #opts{logdir = LogDir}, Args) ->
end
end.
-script_start2(StartOpts = #opts{vts = undefined,
- shell = undefined}, Args) ->
+script_start2(Opts = #opts{vts = undefined,
+ shell = undefined}, Args) ->
case proplists:get_value(spec, Args) of
Specs when Specs =/= [], Specs =/= undefined ->
Specs1 = get_start_opt(join_specs, [Specs], Specs, Args),
@@ -397,38 +397,38 @@ script_start2(StartOpts = #opts{vts = undefined,
{E,Reason} when E == error ; E == 'EXIT' ->
{error,Reason};
TestSpecData ->
- execute_testspecs(TestSpecData, StartOpts, Args, [])
+ execute_all_specs(TestSpecData, Opts, Args, [])
end;
[] ->
{error,no_testspec_specified};
_ -> % no testspec used
%% read config/userconfig from start flags
InitConfig = ct_config:prepare_config_list(Args),
- TheLogDir = which(logdir, StartOpts#opts.logdir),
+ TheLogDir = which(logdir, Opts#opts.logdir),
case check_and_install_configfiles(InitConfig,
TheLogDir,
- StartOpts) of
+ Opts) of
ok -> % go on read tests from start flags
- script_start3(StartOpts#opts{config=InitConfig,
- logdir=TheLogDir}, Args);
+ script_start3(Opts#opts{config=InitConfig,
+ logdir=TheLogDir}, Args);
Error ->
Error
end
end;
-script_start2(StartOpts, Args) ->
+script_start2(Opts, Args) ->
%% read config/userconfig from start flags
InitConfig = ct_config:prepare_config_list(Args),
- LogDir = which(logdir, StartOpts#opts.logdir),
- case check_and_install_configfiles(InitConfig, LogDir, StartOpts) of
+ LogDir = which(logdir, Opts#opts.logdir),
+ case check_and_install_configfiles(InitConfig, LogDir, Opts) of
ok -> % go on read tests from start flags
- script_start3(StartOpts#opts{config=InitConfig,
- logdir=LogDir}, Args);
+ script_start3(Opts#opts{config=InitConfig,
+ logdir=LogDir}, Args);
Error ->
Error
end.
-execute_testspecs([], _, _, Result) ->
+execute_all_specs([], _, _, Result) ->
Result1 = lists:reverse(Result),
case lists:keysearch('EXIT', 1, Result1) of
{value,{_,_,ExitReason}} ->
@@ -447,67 +447,93 @@ execute_testspecs([], _, _, Result) ->
end
end;
-execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) ->
- SpecStartOpts = get_data_for_node(TS, node()),
-
- Label = choose_val(StartOpts#opts.label,
- SpecStartOpts#opts.label),
-
- Profile = choose_val(StartOpts#opts.profile,
- SpecStartOpts#opts.profile),
-
- LogDir = choose_val(StartOpts#opts.logdir,
- SpecStartOpts#opts.logdir),
-
- AllLogOpts = merge_vals([StartOpts#opts.logopts,
- SpecStartOpts#opts.logopts]),
+execute_all_specs([{Specs,TS} | TSs], Opts, Args, Result) ->
+ CombinedOpts = combine_test_opts(TS, Specs, Opts),
+ try execute_one_spec(TS, CombinedOpts, Args) of
+ ExecResult ->
+ execute_all_specs(TSs, Opts, Args, [ExecResult|Result])
+ catch
+ _ : ExitReason ->
+ execute_all_specs(TSs, Opts, Args,
+ [{'EXIT',self(),ExitReason}|Result])
+ end.
+
+execute_one_spec(TS, Opts, Args) ->
+ %% read config/userconfig from start flags
+ InitConfig = ct_config:prepare_config_list(Args),
+ TheLogDir = which(logdir, Opts#opts.logdir),
+ %% merge config from start flags with config from testspec
+ AllConfig = merge_vals([InitConfig, Opts#opts.config]),
+ case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
+ ok -> % read tests from spec
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ do_run(Run, Skip, Opts#opts{config=AllConfig,
+ logdir=TheLogDir}, Args);
+ Error ->
+ Error
+ end.
+
+combine_test_opts(TS, Specs, Opts) ->
+ TSOpts = get_data_for_node(TS, node()),
+
+ Label = choose_val(Opts#opts.label,
+ TSOpts#opts.label),
+
+ Profile = choose_val(Opts#opts.profile,
+ TSOpts#opts.profile),
+
+ LogDir = choose_val(Opts#opts.logdir,
+ TSOpts#opts.logdir),
+
+ AllLogOpts = merge_vals([Opts#opts.logopts,
+ TSOpts#opts.logopts]),
AllVerbosity =
- merge_keyvals([StartOpts#opts.verbosity,
- SpecStartOpts#opts.verbosity]),
+ merge_keyvals([Opts#opts.verbosity,
+ TSOpts#opts.verbosity]),
AllSilentConns =
- merge_vals([StartOpts#opts.silent_connections,
- SpecStartOpts#opts.silent_connections]),
+ merge_vals([Opts#opts.silent_connections,
+ TSOpts#opts.silent_connections]),
Cover =
- choose_val(StartOpts#opts.cover,
- SpecStartOpts#opts.cover),
+ choose_val(Opts#opts.cover,
+ TSOpts#opts.cover),
CoverStop =
- choose_val(StartOpts#opts.cover_stop,
- SpecStartOpts#opts.cover_stop),
+ choose_val(Opts#opts.cover_stop,
+ TSOpts#opts.cover_stop),
MultTT =
- choose_val(StartOpts#opts.multiply_timetraps,
- SpecStartOpts#opts.multiply_timetraps),
+ choose_val(Opts#opts.multiply_timetraps,
+ TSOpts#opts.multiply_timetraps),
ScaleTT =
- choose_val(StartOpts#opts.scale_timetraps,
- SpecStartOpts#opts.scale_timetraps),
-
+ choose_val(Opts#opts.scale_timetraps,
+ TSOpts#opts.scale_timetraps),
+
CreatePrivDir =
- choose_val(StartOpts#opts.create_priv_dir,
- SpecStartOpts#opts.create_priv_dir),
-
+ choose_val(Opts#opts.create_priv_dir,
+ TSOpts#opts.create_priv_dir),
+
AllEvHs =
- merge_vals([StartOpts#opts.event_handlers,
- SpecStartOpts#opts.event_handlers]),
-
+ merge_vals([Opts#opts.event_handlers,
+ TSOpts#opts.event_handlers]),
+
AllCTHooks = merge_vals(
- [StartOpts#opts.ct_hooks,
- SpecStartOpts#opts.ct_hooks]),
-
+ [Opts#opts.ct_hooks,
+ TSOpts#opts.ct_hooks]),
+
EnableBuiltinHooks =
choose_val(
- StartOpts#opts.enable_builtin_hooks,
- SpecStartOpts#opts.enable_builtin_hooks),
-
+ Opts#opts.enable_builtin_hooks,
+ TSOpts#opts.enable_builtin_hooks),
+
Stylesheet =
- choose_val(StartOpts#opts.stylesheet,
- SpecStartOpts#opts.stylesheet),
-
- AllInclude = merge_vals([StartOpts#opts.include,
- SpecStartOpts#opts.include]),
+ choose_val(Opts#opts.stylesheet,
+ TSOpts#opts.stylesheet),
+
+ AllInclude = merge_vals([Opts#opts.include,
+ TSOpts#opts.include]),
application:set_env(common_test, include, AllInclude),
-
+
AutoCompile =
- case choose_val(StartOpts#opts.auto_compile,
- SpecStartOpts#opts.auto_compile) of
+ case choose_val(Opts#opts.auto_compile,
+ TSOpts#opts.auto_compile) of
undefined ->
true;
ACBool ->
@@ -518,8 +544,8 @@ execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) ->
end,
BasicHtml =
- case choose_val(StartOpts#opts.basic_html,
- SpecStartOpts#opts.basic_html) of
+ case choose_val(Opts#opts.basic_html,
+ TSOpts#opts.basic_html) of
undefined ->
false;
BHBool ->
@@ -527,52 +553,27 @@ execute_testspecs([{Specs,TS} | TSs], StartOpts, Args, Result) ->
BHBool),
BHBool
end,
-
- Opts = StartOpts#opts{label = Label,
- profile = Profile,
- testspecs = Specs,
- cover = Cover,
- cover_stop = CoverStop,
- logdir = LogDir,
- logopts = AllLogOpts,
- basic_html = BasicHtml,
- verbosity = AllVerbosity,
- silent_connections = AllSilentConns,
- config = SpecStartOpts#opts.config,
- event_handlers = AllEvHs,
- ct_hooks = AllCTHooks,
- enable_builtin_hooks =
- EnableBuiltinHooks,
- stylesheet = Stylesheet,
- auto_compile = AutoCompile,
- include = AllInclude,
- multiply_timetraps = MultTT,
- scale_timetraps = ScaleTT,
- create_priv_dir = CreatePrivDir},
-
- try execute_testspec(TS, Opts, Args) of
- ExecResult ->
- execute_testspecs(TSs, StartOpts, Args, [ExecResult | Result])
- catch
- _ : ExitReason ->
- execute_testspecs(TSs, StartOpts, Args,
- [{'EXIT',self(),ExitReason} | Result])
- end.
-
-execute_testspec(TS, Opts, Args) ->
- %% read config/userconfig from start flags
- InitConfig = ct_config:prepare_config_list(Args),
- TheLogDir = which(logdir, Opts#opts.logdir),
- %% merge config from start flags with config from testspec
- AllConfig = merge_vals([InitConfig, Opts#opts.config]),
- case check_and_install_configfiles(AllConfig, TheLogDir, Opts) of
- ok -> % read tests from spec
- {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- do_run(Run, Skip, Opts#opts{config=AllConfig,
- logdir=TheLogDir}, Args);
- Error ->
- Error
- end.
+
+ Opts#opts{label = Label,
+ profile = Profile,
+ testspecs = Specs,
+ cover = Cover,
+ cover_stop = CoverStop,
+ logdir = which(logdir, LogDir),
+ logopts = AllLogOpts,
+ basic_html = BasicHtml,
+ verbosity = AllVerbosity,
+ silent_connections = AllSilentConns,
+ config = TSOpts#opts.config,
+ event_handlers = AllEvHs,
+ ct_hooks = AllCTHooks,
+ enable_builtin_hooks = EnableBuiltinHooks,
+ stylesheet = Stylesheet,
+ auto_compile = AutoCompile,
+ include = AllInclude,
+ multiply_timetraps = MultTT,
+ scale_timetraps = ScaleTT,
+ create_priv_dir = CreatePrivDir}.
check_and_install_configfiles(
Configs, LogDir, #opts{
@@ -593,12 +594,12 @@ check_and_install_configfiles(
{error,{cant_load_callback_module,Info}}
end.
-script_start3(StartOpts, Args) ->
- StartOpts1 = get_start_opt(step,
- fun(Step) ->
- StartOpts#opts{step = Step,
- cover = undefined}
- end, StartOpts, Args),
+script_start3(Opts, Args) ->
+ Opts1 = get_start_opt(step,
+ fun(Step) ->
+ Opts#opts{step = Step,
+ cover = undefined}
+ end, Opts, Args),
case {proplists:get_value(dir, Args),
proplists:get_value(suite, Args),
groups_and_cases(proplists:get_value(group, Args),
@@ -612,17 +613,17 @@ script_start3(StartOpts, Args) ->
{error,no_dir_specified};
{Dirs,undefined,[]} when is_list(Dirs) ->
- script_start4(StartOpts#opts{tests = tests(Dirs)}, Args);
+ script_start4(Opts#opts{tests = tests(Dirs)}, Args);
{undefined,Suites,[]} when is_list(Suites) ->
Ts = tests([suite_to_test(S) || S <- Suites]),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
{undefined,Suite,GsAndCs} when is_list(Suite) ->
case [suite_to_test(S) || S <- Suite] of
DirMods = [_] ->
Ts = tests(DirMods, GsAndCs),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
[_,_|_] ->
{error,multiple_suites_and_cases};
_ ->
@@ -636,10 +637,10 @@ script_start3(StartOpts, Args) ->
case [suite_to_test(Dir,S) || S <- Suite] of
DirMods when GsAndCs == [] ->
Ts = tests(DirMods),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
DirMods = [_] when GsAndCs /= [] ->
Ts = tests(DirMods, GsAndCs),
- script_start4(StartOpts1#opts{tests = Ts}, Args);
+ script_start4(Opts1#opts{tests = Ts}, Args);
[_,_|_] when GsAndCs /= [] ->
{error,multiple_suites_and_cases};
_ ->
@@ -650,8 +651,8 @@ script_start3(StartOpts, Args) ->
{error,incorrect_start_options};
{undefined,undefined,_} ->
- if StartOpts#opts.vts ; StartOpts#opts.shell ->
- script_start4(StartOpts#opts{tests = []}, Args);
+ if Opts#opts.vts ; Opts#opts.shell ->
+ script_start4(Opts#opts{tests = []}, Args);
true ->
script_usage(),
{error,missing_start_options}
@@ -781,6 +782,7 @@ script_usage() ->
"\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
"\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]"
"\n\t[-allow_user_terms]"
+ "\n\t[-join_specs]"
"\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
@@ -1075,105 +1077,60 @@ run_test2(StartOpts) ->
end.
run_spec_file(Relaxed,
- Opts = #opts{testspecs = Specs, config = CfgFiles},
+ Opts = #opts{testspecs = Specs},
StartOpts) ->
Specs1 = case Specs of
[X|_] when is_integer(X) -> [Specs];
_ -> Specs
end,
AbsSpecs = lists:map(fun(SF) -> ?abs(SF) end, Specs1),
- log_ts_names(AbsSpecs),
- case catch ct_testspec:collect_tests_from_file(AbsSpecs, Relaxed) of
+ AbsSpecs1 = get_start_opt(join_specs, [AbsSpecs], AbsSpecs, StartOpts),
+ case catch ct_testspec:collect_tests_from_file(AbsSpecs1, Relaxed) of
{Error,CTReason} when Error == error ; Error == 'EXIT' ->
exit({error,CTReason});
- TS ->
- SpecOpts = get_data_for_node(TS, node()),
- Label = choose_val(Opts#opts.label,
- SpecOpts#opts.label),
- Profile = choose_val(Opts#opts.profile,
- SpecOpts#opts.profile),
- LogDir = choose_val(Opts#opts.logdir,
- SpecOpts#opts.logdir),
- AllLogOpts = merge_vals([Opts#opts.logopts,
- SpecOpts#opts.logopts]),
- Stylesheet = choose_val(Opts#opts.stylesheet,
- SpecOpts#opts.stylesheet),
- AllVerbosity = merge_keyvals([Opts#opts.verbosity,
- SpecOpts#opts.verbosity]),
- AllSilentConns = merge_vals([Opts#opts.silent_connections,
- SpecOpts#opts.silent_connections]),
- AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]),
- Cover = choose_val(Opts#opts.cover,
- SpecOpts#opts.cover),
- CoverStop = choose_val(Opts#opts.cover_stop,
- SpecOpts#opts.cover_stop),
- MultTT = choose_val(Opts#opts.multiply_timetraps,
- SpecOpts#opts.multiply_timetraps),
- ScaleTT = choose_val(Opts#opts.scale_timetraps,
- SpecOpts#opts.scale_timetraps),
- CreatePrivDir = choose_val(Opts#opts.create_priv_dir,
- SpecOpts#opts.create_priv_dir),
- AllEvHs = merge_vals([Opts#opts.event_handlers,
- SpecOpts#opts.event_handlers]),
- AllInclude = merge_vals([Opts#opts.include,
- SpecOpts#opts.include]),
- AllCTHooks = merge_vals([Opts#opts.ct_hooks,
- SpecOpts#opts.ct_hooks]),
- EnableBuiltinHooks = choose_val(Opts#opts.enable_builtin_hooks,
- SpecOpts#opts.enable_builtin_hooks),
-
- application:set_env(common_test, include, AllInclude),
-
- AutoCompile = case choose_val(Opts#opts.auto_compile,
- SpecOpts#opts.auto_compile) of
- undefined ->
- true;
- ACBool ->
- application:set_env(common_test, auto_compile,
- ACBool),
- ACBool
- end,
+ TestSpecData ->
+ run_all_specs(TestSpecData, Opts, StartOpts, [])
+ end.
- BasicHtml = case choose_val(Opts#opts.basic_html,
- SpecOpts#opts.basic_html) of
- undefined ->
- false;
- BHBool ->
- application:set_env(common_test, basic_html,
- BHBool),
- BHBool
- end,
-
- Opts1 = Opts#opts{label = Label,
- profile = Profile,
- cover = Cover,
- cover_stop = CoverStop,
- logdir = which(logdir, LogDir),
- logopts = AllLogOpts,
- stylesheet = Stylesheet,
- basic_html = BasicHtml,
- verbosity = AllVerbosity,
- silent_connections = AllSilentConns,
- config = AllConfig,
- event_handlers = AllEvHs,
- auto_compile = AutoCompile,
- include = AllInclude,
- testspecs = AbsSpecs,
- multiply_timetraps = MultTT,
- scale_timetraps = ScaleTT,
- create_priv_dir = CreatePrivDir,
- ct_hooks = AllCTHooks,
- enable_builtin_hooks = EnableBuiltinHooks
- },
-
- case check_and_install_configfiles(AllConfig,Opts1#opts.logdir,
- Opts1) of
- ok ->
- {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
- reformat_result(catch do_run(Run, Skip, Opts1, StartOpts));
- {error,GCFReason} ->
- exit({error,GCFReason})
+run_all_specs([], _, _, TotResult) ->
+ TotResult1 = lists:reverse(TotResult),
+ case lists:keysearch('EXIT', 1, TotResult1) of
+ {value,{_,_,ExitReason}} ->
+ exit(ExitReason);
+ false ->
+ case lists:keysearch(error, 1, TotResult1) of
+ {value,Error} ->
+ Error;
+ false ->
+ lists:foldl(fun({Ok,Fail,{UserSkip,AutoSkip}},
+ {Ok1,Fail1,{UserSkip1,AutoSkip1}}) ->
+ {Ok1+Ok,Fail1+Fail,
+ {UserSkip1+UserSkip,
+ AutoSkip1+AutoSkip}}
+ end, {0,0,{0,0}}, TotResult1)
end
+ end;
+
+run_all_specs([{Specs,TS} | TSs], Opts, StartOpts, TotResult) ->
+ log_ts_names(Specs),
+ Combined = #opts{config = TSConfig} = combine_test_opts(TS, Specs, Opts),
+ AllConfig = merge_vals([Opts#opts.config, TSConfig]),
+ try run_one_spec(TS, Combined#opts{config = AllConfig}, StartOpts) of
+ Result ->
+ run_all_specs(TSs, Opts, StartOpts, [Result | TotResult])
+ catch
+ _ : Reason ->
+ run_all_specs(TSs, Opts, StartOpts, [{error,Reason} | TotResult])
+ end.
+
+run_one_spec(TS, CombinedOpts, StartOpts) ->
+ #opts{logdir = Logdir, config = Config} = CombinedOpts,
+ case check_and_install_configfiles(Config, Logdir, CombinedOpts) of
+ ok ->
+ {Run,Skip} = ct_testspec:prepare_tests(TS, node()),
+ reformat_result(catch do_run(Run, Skip, CombinedOpts, StartOpts));
+ Error ->
+ Error
end.
run_prepared(Run, Skip, Opts = #opts{logdir = LogDir,
@@ -2927,6 +2884,10 @@ opts2args(EnvStartOpts) ->
[{allow_user_terms,[]}];
({allow_user_terms,false}) ->
[];
+ ({join_specs,true}) ->
+ [{join_specs,[]}];
+ ({join_specs,false}) ->
+ [];
({auto_compile,false}) ->
[{no_auto_compile,[]}];
({auto_compile,true}) ->
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 9a7a384ebb..3d9351b71b 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -241,45 +241,71 @@ get_skipped_cases1(_,_,_,[]) ->
%%% collect_tests_from_file reads a testspec file and returns a record
%%% containing the data found.
-collect_tests_from_file(Specs, Relaxed) ->
+collect_tests_from_file(Specs,Relaxed) ->
collect_tests_from_file(Specs,[node()],Relaxed).
collect_tests_from_file(Specs,Nodes,Relaxed) when is_list(Nodes) ->
NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes),
%% [Spec1,Spec2,...] means create one testpec record per Spec file
%% [[Spec1,Spec2,...]] means merge all specs into one testspec record
- {MergeSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)};
+ {JoinSpecs,Specs1} = if is_list(hd(hd(Specs))) -> {true,hd(Specs)};
true -> {false,Specs}
end,
- catch create_specs(Specs1,Specs1,#testspec{nodes=NodeRefs},
- Relaxed,MergeSpecs,[]).
+ TS0 = #testspec{nodes=NodeRefs},
+ %% remove specs without tests
+ Filter = fun({_,#testspec{tests=[]}}) -> false;
+ (_) -> true
+ end,
+ try create_specs(Specs1,TS0,Relaxed,JoinSpecs,{[],TS0},[]) of
+ {{[],_},AdditionalTestSpecs} ->
+ lists:filter(Filter,AdditionalTestSpecs);
+ {{_,#testspec{tests=[]}},AdditionalTestSpecs} ->
+ lists:filter(Filter,AdditionalTestSpecs);
+ {{JoinedSpecs,JoinedTestSpec},AdditionalTestSpecs} ->
+ [{JoinedSpecs,JoinedTestSpec} |
+ lists:filter(Filter,AdditionalTestSpecs)]
+ catch
+ _:Error ->
+ Error
+ end.
-create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) ->
+create_specs([],_,_,_,Joined,Additional) ->
+ {Joined,Additional};
+create_specs([Spec|Ss],TestSpec,Relaxed,JoinSpecs,
+ Joined={JSpecs,_},Additional) ->
SpecDir = filename:dirname(filename:absname(Spec)),
+ TestSpec1 = TestSpec#testspec{spec_dir=SpecDir},
case file:consult(Spec) of
- {ok,Terms} ->
- case collect_tests(Terms,
- TestSpec#testspec{spec_dir=SpecDir},
- Relaxed) of
- TS = #testspec{tests=Tests, logdir=LogDirs} when
- Ss == [], MergeSpecs == true ->
- LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
- [{Specs,TS#testspec{tests=lists:flatten(Tests),
- logdir=LogDirs1}}];
- TS = #testspec{tests=Tests, logdir=LogDirs} when
- Ss == [], MergeSpecs == false ->
- LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
- TSRet = {[Spec],TS#testspec{tests=lists:flatten(Tests),
- logdir=LogDirs1}},
- lists:reverse([TSRet|Saved]);
- TS = #testspec{alias = As, nodes = Ns} when
- MergeSpecs == true ->
- TS1 = TS#testspec{alias = lists:reverse(As),
- nodes = lists:reverse(Ns)},
- create_specs(Ss,Specs,TS1,Relaxed,MergeSpecs,[]);
- TS when MergeSpecs == false ->
- create_specs(Ss,Specs,TestSpec,Relaxed,MergeSpecs,
- [{[Spec],TS}|Saved])
+ {ok,Terms} ->
+ Terms1 = replace_names(Terms),
+ {Specs2Join,Specs2Add} = get_included_specs(Terms1,TestSpec1),
+ TestSpec2 = create_spec(Terms1,TestSpec1,
+ Relaxed,JoinSpecs),
+ case {JoinSpecs,Specs2Join,Specs2Add} of
+ {true,[],[]} ->
+ create_specs(Ss,TestSpec2,Relaxed,JoinSpecs,
+ {JSpecs++[get_absdir(Spec,TestSpec2)],
+ TestSpec2},Additional);
+ {false,[],[]} ->
+ create_specs(Ss,TestSpec,Relaxed,JoinSpecs,Joined,
+ Additional++[{[get_absdir(Spec,TestSpec2)],
+ TestSpec2}]);
+ _ ->
+ {{JSpecs1,JTS1},Additional1} =
+ create_specs(Specs2Join,TestSpec2,Relaxed,true,
+ {[get_absdir(Spec,TestSpec2)],
+ TestSpec2},[]),
+ {Joined2,Additional2} =
+ create_specs(Specs2Add,TestSpec,Relaxed,false,
+ {[],TestSpec1},[]),
+ NewJoined = {JSpecs++JSpecs1,JTS1},
+ NewAdditional = Additional++Additional1++
+ [Joined2 | Additional2],
+ NextTestSpec = if not JoinSpecs -> TestSpec;
+ true -> JTS1
+ end,
+ create_specs(Ss,NextTestSpec,Relaxed,JoinSpecs,
+ NewJoined,NewAdditional)
end;
{error,Reason} ->
ReasonStr =
@@ -287,6 +313,14 @@ create_specs([Spec|Ss],Specs,TestSpec,Relaxed,MergeSpecs,Saved) ->
[file:format_error(Reason)])),
throw({error,{Spec,ReasonStr}})
end.
+
+create_spec(Terms,TestSpec,Relaxed,JoinSpecs) ->
+ TS = #testspec{tests=Tests, logdir=LogDirs} =
+ collect_tests({false,Terms},TestSpec,Relaxed),
+
+ LogDirs1 = lists:delete(".",LogDirs) ++ ["."],
+ TS#testspec{tests=lists:flatten(Tests),
+ logdir=LogDirs1}.
collect_tests_from_list(Terms,Relaxed) ->
collect_tests_from_list(Terms,[node()],Relaxed).
@@ -294,8 +328,8 @@ collect_tests_from_list(Terms,Relaxed) ->
collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) ->
{ok,Cwd} = file:get_cwd(),
NodeRefs = lists:map(fun(N) -> {undefined,N} end, Nodes),
- case catch collect_tests(Terms,#testspec{nodes=NodeRefs,
- spec_dir=Cwd},
+ case catch collect_tests({true,Terms},#testspec{nodes=NodeRefs,
+ spec_dir=Cwd},
Relaxed) of
E = {error,_} ->
E;
@@ -305,10 +339,15 @@ collect_tests_from_list(Terms,Nodes,Relaxed) when is_list(Nodes) ->
TS#testspec{tests=lists:flatten(Tests), logdir=LogDirs1}
end.
-collect_tests(Terms,TestSpec,Relaxed) ->
+collect_tests({Replace,Terms},TestSpec=#testspec{alias=As,nodes=Ns},Relaxed) ->
put(relaxed,Relaxed),
- Terms1 = replace_names(Terms),
- TestSpec1 = get_global(Terms1,TestSpec),
+ Terms1 = if Replace -> replace_names(Terms);
+ true -> Terms
+ end,
+ %% reverse nodes and aliases initially to get the order of them right
+ %% in case this spec is being joined with a previous one
+ TestSpec1 = get_global(Terms1,TestSpec#testspec{alias = lists:reverse(As),
+ nodes = lists:reverse(Ns)}),
TestSpec2 = get_all_nodes(Terms1,TestSpec1),
{Terms2, TestSpec3} = filter_init_terms(Terms1, [], TestSpec2),
add_tests(Terms2,TestSpec3).
@@ -438,9 +477,30 @@ replace_names_in_node1(NodeStr,Defs=[{Name,Replacement}|Ds]) ->
replace_names_in_node1(NodeStr,[]) ->
NodeStr.
+%% look for other specification files, either to join with the
+%% current spec, or execute as additional test runs
+get_included_specs(Terms,TestSpec) ->
+ get_included_specs(Terms,TestSpec,[],[]).
+
+get_included_specs([{specs,How,SpecOrSpecs}|Ts],TestSpec,Join,Add) ->
+ Specs = case SpecOrSpecs of
+ [File|_] when is_list(File) ->
+ [get_absdir(Spec,TestSpec) || Spec <- SpecOrSpecs];
+ [Ch|_] when is_integer(Ch) ->
+ [get_absdir(SpecOrSpecs,TestSpec)]
+ end,
+ if How == join ->
+ get_included_specs(Ts,TestSpec,Join++Specs,Add);
+ true ->
+ get_included_specs(Ts,TestSpec,Join,Add++Specs)
+ end;
+get_included_specs([_|Ts],TestSpec,Join,Add) ->
+ get_included_specs(Ts,TestSpec,Join,Add);
+get_included_specs([],_,Join,Add) ->
+ {Join,Add}.
%% global terms that will be used for analysing all other terms in the spec
-get_global([{merge_tests,Bool} | Ts], Spec) ->
+get_global([{merge_tests,Bool}|Ts],Spec) ->
get_global(Ts,Spec#testspec{merge_tests=Bool});
%% the 'define' term replaces the 'alias' and 'node' terms, but we need to keep
@@ -810,7 +870,10 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled
add_tests([{node,_,_}|Ts],Spec) -> % handled
add_tests(Ts,Spec);
-add_tests([{merge_tests, _} | Ts], Spec) -> % handled
+add_tests([{merge_tests,_} | Ts], Spec) -> % handled
+ add_tests(Ts,Spec);
+
+add_tests([{specs,_,_} | Ts], Spec) -> % handled
add_tests(Ts,Spec);
%% --------------------------------------------------
@@ -1279,6 +1342,7 @@ is_node([],_) ->
valid_terms() ->
[
{define,3},
+ {specs,3},
{node,3},
{cover,2},
{cover,3},
diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl
index 9d2dc84ad3..c150686482 100644
--- a/lib/common_test/test/ct_testspec_2_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_2_SUITE.erl
@@ -479,7 +479,7 @@ multiple_specs(_Config) ->
"multiple_specs.1.spec"),
SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir,
"multiple_specs.2.spec"),
- FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2],
+ FileResult = ct_testspec:collect_tests_from_file([[SpecFile1,SpecFile2]],
false),
ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
@@ -524,7 +524,7 @@ multiple_specs(_Config) ->
%%%
misc_config_terms(_Config) ->
CfgDir = "../cfgs/to1",
-
+ TODir = "../tests/to1",
Spec =
[{node,x,n1@h1},{node,y,n2@h2},
@@ -554,7 +554,9 @@ misc_config_terms(_Config) ->
{create_priv_dir,[auto_per_tc]},
{create_priv_dir,n1@h1,[manual_per_tc]},
- {create_priv_dir,n2@h2,[auto_per_run]}
+ {create_priv_dir,n2@h2,[auto_per_run]},
+
+ {suites,n1@h1,TODir,[x_SUITE]}
],
{ok,SpecDir} = file:get_cwd(),
@@ -599,7 +601,9 @@ misc_config_terms(_Config) ->
{n2@h2,CSS2}],
create_priv_dir = [{Node,[auto_per_tc]},
{n1@h1,[manual_per_tc]},
- {n2@h2,[auto_per_run]}]
+ {n2@h2,[auto_per_run]}],
+ tests = [{{n1@h1,get_absdir(filename:join(SpecDir,TODir))},
+ [{x_SUITE,[all]}]}]
},
verify_result(Verify,ListResult,FileResult).
@@ -688,10 +692,10 @@ define_names_1(_Config) ->
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
-verify_result(Verify,ListResult,FileResult) ->
+verify_result(VerificationRec,ListResult,FileResult) ->
{_,TSLTuples} = rec2proplist(ListResult),
{_,TSFTuples} = rec2proplist(FileResult),
- {_,VTuples} = rec2proplist(Verify),
+ {_,VTuples} = rec2proplist(VerificationRec),
VResult =
(catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) ->
{TSL,TSF};
@@ -720,6 +724,8 @@ read_config(S) ->
rec2proplist(E={error,_What}) ->
exit({invalid_testspec_record,E});
+rec2proplist([{Specs,Rec}]) when is_list(Specs) ->
+ rec2proplist(Rec);
rec2proplist(Rec) ->
[RecName|RecList] = tuple_to_list(Rec),
FieldNames =
generated by cgit v1.2.3 (git 2.25.1) at 2024-12-15 20:28:42 +0000