From 9cf7d07947600ca53e9ace32cdbe4a2554688d51 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 8 Jun 2010 02:55:45 +0200 Subject: Rewrite handling of group definitions in Common Test test suites This is to enable execution of specific groups, and/or test cases within groups, by means of run_test options or test specifications. --- lib/common_test/src/ct_framework.erl | 231 ++++++++++++++++++++++--------- lib/common_test/src/ct_run.erl | 59 ++++++-- lib/test_server/src/test_server_ctrl.erl | 25 +++- 3 files changed, 231 insertions(+), 84 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 3dd1026f13..2a21985fdb 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -29,6 +29,8 @@ -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). +-export([make_conf/5]). + -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -101,7 +103,8 @@ init_tc1(Mod,Func,[Config0],DoInit) when is_list(Config0) -> [{saved_config,{LastFunc,SavedConfig}} | lists:keydelete(saved_config,1,Config0)]; {{LastSuite,InitOrEnd},SavedConfig} when InitOrEnd == init_per_suite ; - InitOrEnd == end_per_suite -> % last suite + InitOrEnd == end_per_suite -> + %% last suite [{saved_config,{LastSuite,SavedConfig}} | lists:keydelete(saved_config,1,Config0)]; undefined -> @@ -649,7 +652,7 @@ get_suite(Mod, all) -> {'EXIT',_} -> get_all(Mod, []); GroupDefs when is_list(GroupDefs) -> - case catch check_groups(Mod, GroupDefs) of + case catch find_groups(Mod, all, all, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly @@ -664,102 +667,178 @@ get_suite(Mod, all) -> %%!============================================================ %%! Note: The handling of sequences in get_suite/2 and get_all/2 -%%! is deprecated and should be removed after OTP R13! +%%! is deprecated and should be removed at some point... %%!============================================================ -get_suite(Mod, Name) -> - %% Name may be name of a group or a test case. If it's a group, - %% it should be expanded to list of cases (in a conf term) +%% group +get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> + Name = proplists:get_value(name, Props), case catch apply(Mod, groups, []) of {'EXIT',_} -> - get_seq(Mod, Name); + [Group]; GroupDefs when is_list(GroupDefs) -> - case catch check_groups(Mod, GroupDefs) of + case catch find_groups(Mod, Name, TCs, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; ConfTests -> - - %%! --- Thu Jun 3 19:13:22 2010 --- peppe was here! - %%! HEERE! - %%! Must be able to search recursively for group Name, - %%! this only handles top level groups! - - FindConf = fun({conf,Props,_,_,_}) -> - case proplists:get_value(name, Props) of - Name -> true; - _ -> false - end - end, - case lists:filter(FindConf, ConfTests) of - [] -> % must be a test case - get_seq(Mod, Name); - [ConfTest|_] -> - ConfTest - end + ConfTests end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}] - end. + end; -check_groups(_Mod, []) -> - []; -check_groups(Mod, Defs) -> - check_groups(Mod, Defs, Defs, []). +%% testcase +get_suite(Mod, Name) -> + get_seq(Mod, Name). -check_groups(Mod, [TC | Gs], Defs, Levels) when is_atom(TC), length(Levels)>0 -> - [TC | check_groups(Mod, Gs, Defs, Levels)]; +%%%----------------------------------------------------------------- -check_groups(Mod, [{group,SubName} | Gs], Defs, Levels) when is_atom(SubName) -> - case lists:member(SubName, Levels) of - true -> - E = "Cyclic reference to group "++atom_to_list(SubName)++ - " in "++atom_to_list(Mod)++":groups/0", - throw({error,list_to_atom(E)}); - false -> - case find_group(Mod, SubName, Defs) of - {error,_} = Error -> - throw(Error); - G -> - [check_groups(Mod, [G], Defs, Levels) | - check_groups(Mod, Gs, Defs, Levels)] - end +find_groups(Mod, Name, TCs, GroupDefs) -> + Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), + Trimmed = trim(Found), + delete_subs(Trimmed, Trimmed). + +find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) -> + cyclic_test(Mod, Name, Known), + [make_conf(Mod, Name, Props, + find(Mod, all, all, Tests, [Name | Known], Defs, true)) | + find(Mod, all, all, Gs, [], Defs, true)]; + +find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false) + when is_atom(Name), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name, Known), + case TCs of + all -> + [make_conf(Mod, Name, Props, + find(Mod, Name, TCs, Tests, [Name | Known], Defs, true))]; + _ -> + Tests1 = [TC || TC <- TCs, + lists:member(TC, Tests) == true], + [make_conf(Mod, Name, Props, Tests1)] end; -check_groups(Mod, [{Name,Tests} | Gs], Defs, Levels) when is_atom(Name), - is_list(Tests) -> - check_groups(Mod, [{Name,[],Tests} | Gs], Defs, Levels); - -check_groups(Mod, [{Name,Props,Tests} | Gs], Defs, Levels) when is_atom(Name), - is_list(Props), - is_list(Tests) -> - {TestSpec,Levels1} = - case Levels of - [] -> - {check_groups(Mod, Tests, Defs, [Name]),[]}; - _ -> - {check_groups(Mod, Tests, Defs, [Name|Levels]),Levels} - end, - [make_conf(Mod, Name, Props, TestSpec) | - check_groups(Mod, Gs, Defs, Levels1)]; +find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false) + when is_atom(Name1), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name1, Known), + [make_conf(Mod, Name1, Props, + find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | + find(Mod, Name, TCs, Gs, [], Defs, false)]; + +find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true) + when is_atom(Name) -> + E = "Duplicate groups named "++atom_to_list(Name)++" in "++ + atom_to_list(Mod)++":groups/0", + throw({error,list_to_atom(E)}); + +find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true) + when is_atom(Name1), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name1, Known), + [make_conf(Mod, Name1, Props, + find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) | + find(Mod, Name, all, Gs, [], Defs, true)]; + +find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) when is_atom(Name1) -> + find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found); + +find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found) + when is_atom(Name1), is_list(Tests) -> + find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found); + +find(Mod, Name, TCs, [TC | Gs], Known, Defs, false) when is_atom(TC) -> + find(Mod, Name, TCs, Gs, Known, Defs, false); + +find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) -> + [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)]; -check_groups(Mod, [BadTerm | _Gs], _Defs, Levels) -> - Where = if length(Levels) == 0 -> +find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) -> + Where = if length(Known) == 0 -> atom_to_list(Mod)++":groups/0"; true -> - "group "++atom_to_list(lists:last(Levels))++ + "group "++atom_to_list(lists:last(Known))++ " in "++atom_to_list(Mod)++":groups/0" end, Term = io_lib:format("~p", [BadTerm]), E = "Bad term "++lists:flatten(Term)++" in "++Where, throw({error,list_to_atom(E)}); -check_groups(_Mod, [], _Defs, _) -> +find(_Mod, _Name, _TCs, [], _Known, _Defs, false) -> + ['$NOMATCH']; + +find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) -> []. -find_group(Mod, Name, Defs) -> +delete_subs([Conf | Confs], All) -> + All1 = delete_conf(Conf, All), + case is_sub(Conf, All1) of + true -> + delete_subs(Confs, All1); + false -> + delete_subs(Confs, All) + end; + +delete_subs([], All) -> + All. + +delete_conf({conf,Props,_,_,_}, Confs) -> + Name = proplists:get_value(name, Props), + [Conf || Conf = {conf,Props0,_,_,_} <- Confs, + Name =/= proplists:get_value(name, Props0)]. + +is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> + Name = proplists:get_value(name, Props), + case lists:any(fun({conf,Props0,_,_,_}) -> + case proplists:get_value(name, Props0) of + N when N == Name -> + true; + _ -> + false + end; + (_) -> + false + end, Tests) of + true -> + true; + false -> + is_sub(Conf, Tests) or is_sub(Conf, Confs) + end; + +is_sub(Conf, [_TC | Tests]) -> + is_sub(Conf, Tests); + +is_sub(_Conf, []) -> + false. + +trim(['$NOMATCH' | Tests]) -> + trim(Tests); + +trim([{conf,Props,Init,Tests,End} | Confs]) -> + case trim(Tests) of + [] -> + trim(Confs); + Trimmed -> + [{conf,Props,Init,Trimmed,End} | trim(Confs)] + end; + +trim([TC | Tests]) -> + [TC | trim(Tests)]; + +trim([]) -> + []. + +cyclic_test(Mod, Name, Names) -> + case lists:member(Name, Names) of + true -> + E = "Cyclic reference to group "++atom_to_list(Name)++ + " in "++atom_to_list(Mod)++":groups/0", + throw({error,list_to_atom(E)}); + false -> + ok + end. + +expand(Mod, Name, Defs) -> case lists:keysearch(Name, 1, Defs) of {value,Def} -> Def; @@ -769,7 +848,22 @@ find_group(Mod, Name, Defs) -> throw({error,list_to_atom(E)}) end. +make_conf(Dir, Mod, Name, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_conf(Mod, Name, Props, TestSpec). + make_conf(Mod, Name, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_file(Mod); + _ -> + ok + end, {InitConf,EndConf} = case erlang:function_exported(Mod,init_per_group,2) of true -> @@ -780,6 +874,7 @@ make_conf(Mod, Name, Props, TestSpec) -> end, {conf,[{name,Name}|Props],InitConf,TestSpec,EndConf}. +%%%----------------------------------------------------------------- get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 6a9c42d1b9..28fc5ceb74 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1354,17 +1354,31 @@ final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) when Cases==[]; Cases==all -> final_tests([{TestDir,[Suite],all}|Tests], Final, Skip, Bad); -final_tests([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) -> +final_tests([{TestDir,Suite,Groups}|Tests], Final, Skip, Bad) when + is_atom(element(1,hd(Groups))) -> + Confs = + lists:map(fun({Group,TCs}) -> + ct_framework:make_conf(TestDir, Suite, + Group, [], TCs) + end, Groups), + Do = {TestDir,Suite,Confs}, case lists:keymember({TestDir,Suite}, 1, Bad) of false -> - Do = {TestDir,Suite,Cases}, final_tests(Tests, [Do|Final], Skip, Bad); true -> - Do = {TestDir,Suite,Cases}, - Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}], + Skip1 = Skip ++ [{TestDir,Suite,Confs,"Make failed"}], final_tests(Tests, [Do|Final], Skip1, Bad) end; +final_tests([Do={TestDir,Suite,Cases}|Tests], Final, Skip, Bad) -> + case lists:keymember({TestDir,Suite}, 1, Bad) of + true -> + Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}], + final_tests(Tests, [Do|Final], Skip1, Bad); + false -> + final_tests(Tests, [Do|Final], Skip, Bad) + end; + final_tests([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. @@ -1604,13 +1618,36 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> Error end; -%% group -add_jobs([{TestDir,Suite,[{GroupName,_Cases}]}|Tests], Skip, Opts, CleanUp) when - is_atom(GroupName) -> - add_jobs([{TestDir,Suite,GroupName}|Tests], Skip, Opts, CleanUp); -add_jobs([{TestDir,Suite,{GroupName,_Cases}}|Tests], Skip, Opts, CleanUp) when - is_atom(GroupName) -> - add_jobs([{TestDir,Suite,GroupName}|Tests], Skip, Opts, CleanUp); +%% group (= conf case in test_server) +add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when + element(1, hd(Confs)) == conf -> + Group = fun(Conf) -> proplists:get_value(name, element(2, Conf)) end, + TestCases = fun(Conf) -> element(4, Conf) end, + TCTestName = fun(all) -> ""; + ([C]) when is_atom(C) -> "." ++ atom_to_list(C); + (Cs) when is_list(Cs) -> ".cases" + end, + GrTestName = + case Confs of + [Conf] -> + "." ++ atom_to_list(Group(Conf)) ++ TCTestName(TestCases(Conf)); + _ -> + ".groups" + end, + TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName, + case maybe_interpret(Suite, init_per_group, Opts) of + ok -> + case catch test_server_ctrl:add_conf_with_skip(TestName, Suite, Confs, + skiplist(TestDir,Skip)) of + {'EXIT',_} -> + CleanUp; + _ -> + wait_for_idle(), + add_jobs(Tests, Skip, Opts, [Suite|CleanUp]) + end; + Error -> + Error + end; %% test case add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -> diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 1245c10a01..d1f53f3c39 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -151,10 +151,12 @@ %%% OPERATOR INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([add_spec/1, add_dir/2, add_dir/3]). --export([add_module/1, add_module/2, add_case/2, add_case/3, add_cases/2, - add_cases/3]). +-export([add_module/1, add_module/2, + add_conf/3, + add_case/2, add_case/3, add_cases/2, add_cases/3]). -export([add_dir_with_skip/3, add_dir_with_skip/4, add_tests_with_skip/3]). -export([add_module_with_skip/2, add_module_with_skip/3, + add_conf_with_skip/4, add_case_with_skip/3, add_case_with_skip/4, add_cases_with_skip/3, add_cases_with_skip/4]). -export([jobs/0, run_test/1, wait_finish/0, idle_notify/1, @@ -236,9 +238,16 @@ add_dir(Name, Dir, Pattern) -> add_module(Mod) when is_atom(Mod) -> add_job(atom_to_list(Mod), {Mod,all}). + add_module(Name, Mods) when is_list(Mods) -> add_job(cast_to_list(Name), lists:map(fun(Mod) -> {Mod,all} end, Mods)). +add_conf(Name, Mod, Conf) when is_tuple(Conf) -> + add_job(cast_to_list(Name), {Mod,[Conf]}); + +add_conf(Name, Mod, Confs) when is_list(Confs) -> + add_job(cast_to_list(Name), {Mod,Confs}). + add_case(Mod, Case) when is_atom(Mod), is_atom(Case) -> add_job(atom_to_list(Mod), {Mod,Case}). @@ -283,6 +292,12 @@ add_module_with_skip(Mod, Skip) when is_atom(Mod) -> add_module_with_skip(Name, Mods, Skip) when is_list(Mods) -> add_job(cast_to_list(Name), lists:map(fun(Mod) -> {Mod,all} end, Mods), Skip). +add_conf_with_skip(Name, Mod, Conf, Skip) when is_tuple(Conf) -> + add_job(cast_to_list(Name), {Mod,[Conf]}, Skip); + +add_conf_with_skip(Name, Mod, Confs, Skip) when is_list(Confs) -> + add_job(cast_to_list(Name), {Mod,Confs}, Skip). + add_case_with_skip(Mod, Case, Skip) when is_atom(Mod), is_atom(Case) -> add_job(atom_to_list(Mod), {Mod,Case}, Skip). @@ -1549,7 +1564,7 @@ temp_nodename([Chr|Base], Acc) -> %% of cases can not be calculated and NoOfCases = unknown. count_test_cases(TopCases, SkipCases) when is_list(TopCases) -> case collect_all_cases(TopCases, SkipCases) of - {error,_} -> + {error,_Why} -> error; TestSpec -> {get_suites(TestSpec, []), @@ -4517,8 +4532,8 @@ collect_cases({_Mod,_Case,_Args}=Spec, St) -> collect_case(Spec, St); collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) -> collect_case({St#cc.mod,Case}, St); -collect_cases(Other, _St) -> - {error,{bad_subtest_spec,Other}}. +collect_cases(Other, St) -> + {error,{bad_subtest_spec,St#cc.mod,Other}}. collect_case(MFA, St) -> case in_skip_list(MFA, St#cc.skip) of -- cgit v1.2.3 From 5737182da70d4f07a15e8d50197dc40251f615f6 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Sat, 19 Jun 2010 00:03:01 +0200 Subject: Fix problem with using the wrong logdir setting The default logdir value "." overrides setting in test specification if logdir start flag/option is missing. --- lib/common_test/src/ct_run.erl | 59 ++++++++++++---------- .../groups_1/test/groups_11_SUITE.erl | 2 + 2 files changed, 34 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 28fc5ceb74..b6ead90a8c 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -160,7 +160,7 @@ script_start1(Parent, Args) -> Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), - LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, ".", Args), + LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), MultTT = get_start_opt(multiply_timetraps, fun([MT]) -> list_to_integer(MT) end, 1, Args), ScaleTT = get_start_opt(scale_timetraps, @@ -317,29 +317,30 @@ script_start2(StartOpts = #opts{vts = undefined, end, %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), + TheLogDir = which(logdir, Opts#opts.logdir), case {TestSpec,Terms} of {_,{error,_}=Error} -> Error; {[],_} -> {error,no_testspec_specified}; {undefined,_} -> % no testspec used - case check_and_install_configfiles(InitConfig, - which(logdir,Opts#opts.logdir), + case check_and_install_configfiles(InitConfig, TheLogDir, Opts#opts.event_handlers) of ok -> % go on read tests from start flags - script_start3(Opts#opts{config=InitConfig}, Args); + script_start3(Opts#opts{config=InitConfig, + logdir=TheLogDir}, Args); Error -> Error end; {_,_} -> % testspec used %% merge config from start flags with config from testspec AllConfig = merge_vals([InitConfig, Opts#opts.config]), - case check_and_install_configfiles(AllConfig, - which(logdir,Opts#opts.logdir), + case check_and_install_configfiles(AllConfig, TheLogDir, Opts#opts.event_handlers) of ok -> % read tests from spec {Run,Skip} = ct_testspec:prepare_tests(Terms, node()), - do_run(Run, Skip, Opts#opts{config=AllConfig}, Args); + do_run(Run, Skip, Opts#opts{config=AllConfig, + logdir=TheLogDir}, Args); Error -> Error end @@ -348,11 +349,12 @@ script_start2(StartOpts = #opts{vts = undefined, script_start2(StartOpts, Args) -> %% read config/userconfig from start flags InitConfig = ct_config:prepare_config_list(Args), - case check_and_install_configfiles(InitConfig, - which(logdir,StartOpts#opts.logdir), + LogDir = which(logdir, StartOpts#opts.logdir), + case check_and_install_configfiles(InitConfig, LogDir, StartOpts#opts.event_handlers) of ok -> % go on read tests from start flags - script_start3(StartOpts#opts{config=InitConfig}, Args); + script_start3(StartOpts#opts{config=InitConfig, + logdir=LogDir}, Args); Error -> Error end. @@ -615,7 +617,7 @@ run_test(StartOpts) when is_list(StartOpts) -> run_test1(StartOpts) -> %% logdir LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end, - ".", StartOpts), + StartOpts), %% config & userconfig CfgFiles = ct_config:get_config_file_list(StartOpts), @@ -767,7 +769,7 @@ run_spec_file(Relaxed, AllEvHs) of ok -> Opts1 = Opts#opts{cover = Cover, - logdir = LogDir, + logdir = which(logdir, LogDir), config = AllConfig, event_handlers = AllEvHs, include = AllInclude, @@ -785,9 +787,10 @@ run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, config = CfgFiles, event_handlers = EvHandlers}, StartOpts) -> - case check_and_install_configfiles(CfgFiles, LogDir, EvHandlers) of + LogDir1 = which(logdir, LogDir), + case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of ok -> - do_run(Run, Skip, Opts, StartOpts); + do_run(Run, Skip, Opts#opts{logdir = LogDir1}, StartOpts); {error,Reason} -> exit(Reason) end. @@ -816,6 +819,8 @@ check_config_file(Callback, File)-> run_dir(Opts = #opts{logdir = LogDir, config = CfgFiles, event_handlers = EvHandlers}, StartOpts) -> + LogDir1 = which(logdir, LogDir), + Opts1 = Opts#opts{logdir = LogDir1}, AbsCfgFiles = lists:map(fun({Callback,FileList})-> case code:is_loaded(Callback) of @@ -834,7 +839,7 @@ run_dir(Opts = #opts{logdir = LogDir, check_config_file(Callback, File) end, FileList)} end, CfgFiles), - case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir) of + case install([{config,AbsCfgFiles},{event_handler,EvHandlers}], LogDir1) of ok -> ok; {error,IReason} -> exit(IReason) end, @@ -842,7 +847,7 @@ run_dir(Opts = #opts{logdir = LogDir, {value,{_,Dirs=[Dir|_]}} when not is_integer(Dir), length(Dirs)>1 -> %% multiple dirs (no suite) - do_run(tests(Dirs), [], Opts, StartOpts); + do_run(tests(Dirs), [], Opts1, StartOpts); false -> % no dir %% fun for converting suite name to {Dir,Mod} tuple S2M = fun(S) when is_list(S) -> @@ -857,12 +862,12 @@ run_dir(Opts = #opts{logdir = LogDir, case listify(proplists:get_value(group, StartOpts, [])) ++ listify(proplists:get_value(testcase, StartOpts, [])) of [] -> - do_run(tests(Dir, listify(Mod)), [], Opts, StartOpts); + do_run(tests(Dir, listify(Mod)), [], Opts1, StartOpts); GsAndCs -> - do_run(tests(Dir, Mod, GsAndCs), [], Opts, StartOpts) + do_run(tests(Dir, Mod, GsAndCs), [], Opts1, StartOpts) end; {value,{_,Suites}} -> - do_run(tests(lists:map(S2M, Suites)), [], Opts, StartOpts); + do_run(tests(lists:map(S2M, Suites)), [], Opts1, StartOpts); _ -> exit(no_tests_specified) end; @@ -875,17 +880,17 @@ run_dir(Opts = #opts{logdir = LogDir, case listify(proplists:get_value(group, StartOpts, [])) ++ listify(proplists:get_value(testcase, StartOpts, [])) of [] -> - do_run(tests(Dir, listify(Mod)), [], Opts, StartOpts); + do_run(tests(Dir, listify(Mod)), [], Opts1, StartOpts); GsAndCs -> - do_run(tests(Dir, Mod, GsAndCs), [], Opts, StartOpts) + do_run(tests(Dir, Mod, GsAndCs), [], Opts1, StartOpts) end; {value,{_,Suites=[Suite|_]}} when is_list(Suite) -> Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites), - do_run(tests(delistify(Dir), Mods), [], Opts, StartOpts); + do_run(tests(delistify(Dir), Mods), [], Opts1, StartOpts); {value,{_,Suites}} -> - do_run(tests(delistify(Dir), Suites), [], Opts, StartOpts); + do_run(tests(delistify(Dir), Suites), [], Opts1, StartOpts); false -> % no suite, only dir - do_run(tests(listify(Dir)), [], Opts, StartOpts) + do_run(tests(listify(Dir)), [], Opts1, StartOpts) end end. @@ -925,12 +930,12 @@ run_testspec1(TestSpec) -> EnvInclude++Opts#opts.include end, application:set_env(common_test, include, AllInclude), - - case check_and_install_configfiles(Opts#opts.config, - which(logdir,Opts#opts.logdir), + LogDir1 = which(logdir,Opts#opts.logdir), + case check_and_install_configfiles(Opts#opts.config, LogDir1, Opts#opts.event_handlers) of ok -> Opts1 = Opts#opts{testspecs = [TestSpec], + logdir = LogDir1, include = AllInclude}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), do_run(Run, Skip, Opts1, []); diff --git a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl index c6d50443d0..c69400e938 100644 --- a/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_1_SUITE_data/groups_1/test/groups_11_SUITE.erl @@ -99,6 +99,7 @@ init_per_group(Group, Config) -> {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of true -> + ct:comment(Group), init = ?config(suite,Config), [{Group,Group} | Config]; false -> @@ -109,6 +110,7 @@ end_per_group(Group, Config) -> {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of true -> + ct:comment(Group), init = ?config(suite,Config), Group = ?config(Group,Config), ok; -- cgit v1.2.3 From 4b399cdf4472975b0f3d4d21054f30064b32ed32 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 22 Jun 2010 17:15:00 +0200 Subject: Implement handling of group specifications It's possible now to specify execution of test case groups (and test cases within groups) by means of run_test options and test specification terms. From test specifications, groups (and test cases) may also be skipped. ct_run has been improved to report errors better. New test suites have been introduced. --- lib/common_test/src/ct_framework.erl | 2 + lib/common_test/src/ct_run.erl | 163 +++++++---- lib/common_test/src/ct_testspec.erl | 20 +- lib/common_test/test/Makefile | 1 + lib/common_test/test/ct_groups_test_2_SUITE.erl | 24 +- .../specs/groups_2.1.spec | 26 -- lib/common_test/test/ct_test_support.erl | 2 +- lib/common_test/test/ct_testspec_1_SUITE.erl | 209 ++++++++++++++ .../groups_1/groups_11_SUITE.erl | 281 ++++++++++++++++++ .../groups_1/groups_12_SUITE.erl | 314 +++++++++++++++++++++ .../groups_2/groups_21_SUITE.erl | 281 ++++++++++++++++++ .../groups_2/groups_22_SUITE.erl | 314 +++++++++++++++++++++ .../suites_1/simple_1_SUITE.erl | 146 ++++++++++ .../suites_1/simple_2_SUITE.erl | 146 ++++++++++ lib/test_server/src/test_server_ctrl.erl | 110 ++++++-- 15 files changed, 1904 insertions(+), 135 deletions(-) delete mode 100644 lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec create mode 100644 lib/common_test/test/ct_testspec_1_SUITE.erl create mode 100644 lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl create mode 100644 lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl create mode 100644 lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl create mode 100644 lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl create mode 100644 lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl create mode 100644 lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 2a21985fdb..25915b9db2 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -682,6 +682,8 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; + [] -> + {error,{invalid_group_spec,Name}}; ConfTests -> ConfTests end; diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index b6ead90a8c..adb49aa44d 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -153,6 +153,7 @@ script_start(Args) -> Result end, stop_trace(Tracing), + timer:sleep(1000), Res. script_start1(Parent, Args) -> @@ -777,7 +778,7 @@ run_spec_file(Relaxed, multiply_timetraps = MultTT, scale_timetraps = ScaleTT}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, Opts1, StartOpts); + reformat_result(catch do_run(Run, Skip, Opts1, StartOpts)); {error,GCFReason} -> exit(GCFReason) end @@ -790,7 +791,8 @@ run_prepared(Run, Skip, Opts = #opts{logdir = LogDir, LogDir1 = which(logdir, LogDir), case check_and_install_configfiles(CfgFiles, LogDir1, EvHandlers) of ok -> - do_run(Run, Skip, Opts#opts{logdir = LogDir1}, StartOpts); + reformat_result(catch do_run(Run, Skip, Opts#opts{logdir = LogDir1}, + StartOpts)); {error,Reason} -> exit(Reason) end. @@ -847,7 +849,7 @@ run_dir(Opts = #opts{logdir = LogDir, {value,{_,Dirs=[Dir|_]}} when not is_integer(Dir), length(Dirs)>1 -> %% multiple dirs (no suite) - do_run(tests(Dirs), [], Opts1, StartOpts); + reformat_result(catch do_run(tests(Dirs), [], Opts1, StartOpts)); false -> % no dir %% fun for converting suite name to {Dir,Mod} tuple S2M = fun(S) when is_list(S) -> @@ -862,12 +864,15 @@ run_dir(Opts = #opts{logdir = LogDir, case listify(proplists:get_value(group, StartOpts, [])) ++ listify(proplists:get_value(testcase, StartOpts, [])) of [] -> - do_run(tests(Dir, listify(Mod)), [], Opts1, StartOpts); + reformat_result(catch do_run(tests(Dir, listify(Mod)), + [], Opts1, StartOpts)); GsAndCs -> - do_run(tests(Dir, Mod, GsAndCs), [], Opts1, StartOpts) + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)) end; {value,{_,Suites}} -> - do_run(tests(lists:map(S2M, Suites)), [], Opts1, StartOpts); + reformat_result(catch do_run(tests(lists:map(S2M, Suites)), + [], Opts1, StartOpts)); _ -> exit(no_tests_specified) end; @@ -880,17 +885,22 @@ run_dir(Opts = #opts{logdir = LogDir, case listify(proplists:get_value(group, StartOpts, [])) ++ listify(proplists:get_value(testcase, StartOpts, [])) of [] -> - do_run(tests(Dir, listify(Mod)), [], Opts1, StartOpts); + reformat_result(catch do_run(tests(Dir, listify(Mod)), + [], Opts1, StartOpts)); GsAndCs -> - do_run(tests(Dir, Mod, GsAndCs), [], Opts1, StartOpts) + reformat_result(catch do_run(tests(Dir, Mod, GsAndCs), + [], Opts1, StartOpts)) end; {value,{_,Suites=[Suite|_]}} when is_list(Suite) -> Mods = lists:map(fun(Str) -> list_to_atom(Str) end, Suites), - do_run(tests(delistify(Dir), Mods), [], Opts1, StartOpts); + reformat_result(catch do_run(tests(delistify(Dir), Mods), + [], Opts1, StartOpts)); {value,{_,Suites}} -> - do_run(tests(delistify(Dir), Suites), [], Opts1, StartOpts); + reformat_result(catch do_run(tests(delistify(Dir), Suites), + [], Opts1, StartOpts)); false -> % no suite, only dir - do_run(tests(listify(Dir)), [], Opts1, StartOpts) + reformat_result(catch do_run(tests(listify(Dir)), + [], Opts1, StartOpts)) end end. @@ -934,11 +944,11 @@ run_testspec1(TestSpec) -> case check_and_install_configfiles(Opts#opts.config, LogDir1, Opts#opts.event_handlers) of ok -> - Opts1 = Opts#opts{testspecs = [TestSpec], + Opts1 = Opts#opts{testspecs = [], logdir = LogDir1, include = AllInclude}, {Run,Skip} = ct_testspec:prepare_tests(TS, node()), - do_run(Run, Skip, Opts1, []); + reformat_result(catch do_run(Run, Skip, Opts1, [])); {error,GCFReason} -> exit(GCFReason) end @@ -1028,21 +1038,26 @@ delistify(E) -> E. %%% @equiv ct:run/3 run(TestDir, Suite, Cases) -> install([]), - do_run(tests(TestDir, Suite, Cases), []). + reformat_result(catch do_run(tests(TestDir, Suite, Cases), [])). %%%----------------------------------------------------------------- %%% @hidden %%% @equiv ct:run/2 run(TestDir, Suite) when is_list(TestDir), is_integer(hd(TestDir)) -> install([]), - do_run(tests(TestDir, Suite), []). + reformat_result(catch do_run(tests(TestDir, Suite), [])). %%%----------------------------------------------------------------- %%% @hidden %%% @equiv ct:run/1 run(TestDirs) -> install([]), - do_run(tests(TestDirs), []). + reformat_result(catch do_run(tests(TestDirs), [])). + +reformat_result({user_error,Reason}) -> + {error,Reason}; +reformat_result(Result) -> + Result. suite_to_test(Suite) -> {filename:dirname(Suite),list_to_atom(filename:rootname(filename:basename(Suite)))}. @@ -1161,10 +1176,19 @@ do_run(Tests, Skip, Opts, Args) -> true -> SavedErrors = save_make_errors(SuiteMakeErrors), ct_repeat:log_loop_info(Args), - {Tests1,Skip1} = final_tests(Tests,[],Skip,SavedErrors), - R = do_run_test(Tests1, Skip1, Opts1), - ct_util:stop(normal), - R; + + {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors), + + R = (catch do_run_test(Tests1, Skip1, Opts1)), + case R of + {EType,_} = Error when EType == user_error ; + EType == error -> + ct_util:stop(clean), + exit(Error); + _ -> + ct_util:stop(normal), + R + end; false -> io:nl(), ct_util:stop(clean), @@ -1321,8 +1345,20 @@ suite_tuples([{TestDir,Suite,_} | Tests]) when is_atom(Suite) -> suite_tuples([]) -> []. -final_tests([{TestDir,Suites,_}|Tests], - Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) -> +final_tests(Tests, Skip, Bad) -> + +%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! +io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]), + + {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad), + +%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! +io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip1]), + + {Tests1,final_skip(Skip1, [])}. + +final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when + is_list(Suites), is_atom(hd(Suites)) -> % Separate = % fun(S,{DoSuite,Dont}) -> % case lists:keymember({TestDir,S},1,Bad) of @@ -1341,9 +1377,9 @@ final_tests([{TestDir,Suites,_}|Tests], Skip1 = [{TD,S,"Make failed"} || {{TD,S},_} <- Bad, S1 <- Suites, S == S1, TD == TestDir], Final1 = [{TestDir,S,all} || S <- Suites], - final_tests(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad); + final_tests1(Tests, lists:reverse(Final1)++Final, Skip++Skip1, Bad); -final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) -> +final_tests1([{TestDir,all,all}|Tests], Final, Skip, Bad) -> MissingSuites = case lists:keysearch({TestDir,all}, 1, Bad) of {value,{_,Failed}} -> @@ -1353,40 +1389,47 @@ final_tests([{TestDir,all,all}|Tests], Final, Skip, Bad) -> end, Missing = [{TestDir,S,"Make failed"} || S <- MissingSuites], Final1 = [{TestDir,all,all}|Final], - final_tests(Tests, Final1, Skip++Missing, Bad); + final_tests1(Tests, Final1, Skip++Missing, Bad); -final_tests([{TestDir,Suite,Cases}|Tests], - Final, Skip, Bad) when Cases==[]; Cases==all -> - final_tests([{TestDir,[Suite],all}|Tests], Final, Skip, Bad); +final_tests1([{TestDir,Suite,Cases}|Tests], Final, Skip, Bad) when + Cases==[]; Cases==all -> + final_tests1([{TestDir,[Suite],all}|Tests], Final, Skip, Bad); -final_tests([{TestDir,Suite,Groups}|Tests], Final, Skip, Bad) when - is_atom(element(1,hd(Groups))) -> - Confs = - lists:map(fun({Group,TCs}) -> - ct_framework:make_conf(TestDir, Suite, - Group, [], TCs) - end, Groups), - Do = {TestDir,Suite,Confs}, +final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when + is_list(GrsOrCs) -> case lists:keymember({TestDir,Suite}, 1, Bad) of - false -> - final_tests(Tests, [Do|Final], Skip, Bad); true -> - Skip1 = Skip ++ [{TestDir,Suite,Confs,"Make failed"}], - final_tests(Tests, [Do|Final], Skip1, Bad) - end; - -final_tests([Do={TestDir,Suite,Cases}|Tests], Final, Skip, Bad) -> - case lists:keymember({TestDir,Suite}, 1, Bad) of - true -> - Skip1 = Skip ++ [{TestDir,Suite,Cases,"Make failed"}], - final_tests(Tests, [Do|Final], Skip1, Bad); + Skip1 = Skip ++ [{TestDir,Suite,all,"Make failed"}], + final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad); false -> - final_tests(Tests, [Do|Final], Skip, Bad) + GrsOrCs1 = + lists:map( + %% for now, only flat group defs are allowed as + %% start options and test spec terms + fun({Group,TCs}) -> + ct_framework:make_conf(TestDir, Suite, + Group, [], TCs); + (TC) -> + TC + end, GrsOrCs), + Do = {TestDir,Suite,GrsOrCs1}, + final_tests1(Tests, [Do|Final], Skip, Bad) end; -final_tests([], Final, Skip, _Bad) -> +final_tests1([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. +final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> + Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs), + Skip = {TestDir,Suite,Conf,Reason}, + final_skip(Skips, [Skip|Final]); + +final_skip([Skip|Skips], Final) -> + final_skip(Skips, [Skip|Final]); + +final_skip([], Final) -> + lists:reverse(Final). + continue([]) -> true; continue(_MakeErrors) -> @@ -1508,6 +1551,7 @@ do_run_test(Tests, Skip, Opts) -> _ -> false end, + %% let test_server expand the test tuples and count no of cases {Suites,NoOfCases} = count_test_cases(Tests, Skip), Suites1 = delete_dups(Suites), @@ -1563,19 +1607,30 @@ count_test_cases(Tests, Skip) -> TSPid = test_server_ctrl:start_get_totals(SendResult), Ref = erlang:monitor(process, TSPid), add_jobs(Tests, Skip, #opts{}, []), - {Suites,NoOfCases} = count_test_cases1(length(Tests), 0, [], Ref), + Counted = (catch count_test_cases1(length(Tests), 0, [], Ref)), erlang:demonitor(Ref, [flush]), - test_server_ctrl:stop_get_totals(), - {Suites,NoOfCases}. + case Counted of + {error,{test_server_died}} = Error -> + throw(Error); + {error,Reason} -> + unlink(whereis(test_server_ctrl)), + test_server_ctrl:stop(), + throw({user_error,Reason}); + Result -> + test_server_ctrl:stop_get_totals(), + Result + end. count_test_cases1(0, N, Suites, _) -> {lists:flatten(Suites), N}; count_test_cases1(Jobs, N, Suites, Ref) -> receive + {_,{error,_Reason} = Error} -> + throw(Error); {no_of_cases,{Ss,N1}} -> count_test_cases1(Jobs-1, add_known(N,N1), [Ss|Suites], Ref); - {'DOWN', Ref, _, _, _} -> - {[],0} + {'DOWN', Ref, _, _, Info} -> + throw({error,{test_server_died,Info}}) end. add_known(unknown, _) -> diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 0f68b062f6..9d2a791406 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -185,7 +185,15 @@ prepare_cases(Node,Dir,Suite,Cases) -> {[{{Node,Dir},{Suite,all}}],SkipAll}; Skipped -> %% note: this adds a test even if only skip is specified - PrepC = lists:foldr(fun({C,{skip,_Cmt}},Acc) -> + PrepC = lists:foldr(fun({{G,_}=Group,{skip,_Cmt}}, Acc) when + is_atom(G) -> + case lists:keymember(G, 1, Cases) of + true -> + Acc; + false -> + [Group|Acc] + end; + ({C,{skip,_Cmt}},Acc) -> case lists:member(C,Cases) of true -> Acc; @@ -194,7 +202,7 @@ prepare_cases(Node,Dir,Suite,Cases) -> end; (C,Acc) -> [C|Acc] end, [], Cases), - {{{Node,Dir},{Suite,PrepC}},Skipped} + {{{Node,Dir},{Suite,PrepC}},Skipped} end. get_skipped_suites(Node,Dir,Suites) -> @@ -878,8 +886,12 @@ skip_suites(_Node,_Dir,[],_Cmt,Tests) -> skip_suites(Node,Dir,S,Cmt,Tests) -> skip_suites(Node,Dir,[S],Cmt,Tests). -skip_groups(Node,Dir,Suite,Group,Case,Cmt,Tests) when is_atom(Group) -> - skip_groups(Node,Dir,Suite,[Group],[Case],Cmt,Tests); +skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); +skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> + skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> + skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> Suites = diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 97ded5eb9a..3fb0d627a0 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -32,6 +32,7 @@ MODULES= \ ct_event_handler_SUITE \ ct_groups_test_1_SUITE \ ct_groups_test_2_SUITE \ + ct_testspec_1_SUITE \ ct_skip_SUITE \ ct_error_SUITE \ ct_test_server_if_1_SUITE \ diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl index 56e0ac30c7..c4371501b3 100644 --- a/lib/common_test/test/ct_groups_test_2_SUITE.erl +++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl @@ -60,7 +60,7 @@ all(doc) -> ["Run smoke tests of Common Test."]; all(suite) -> - [missing_conf, testspec_1, repeat_1]. + [missing_conf, repeat_1]. %%-------------------------------------------------------------------- %% TEST CASES @@ -88,25 +88,6 @@ missing_conf(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -testspec_1(Config) when is_list(Config) -> - DataDir = ?config(data_dir, Config), - - TestSpec = filename:join(DataDir, "specs/groups_2.1.spec"), - - {Opts,ERPid} = setup({spec,TestSpec}, Config), - ok = ct_test_support:run(Opts, Config), - Events = ct_test_support:get_events(ERPid, Config), - - ct_test_support:log_events(testspec_1, - reformat(Events, ?eh), - ?config(priv_dir, Config)), - - TestEvents = events_to_check(testspec_1), - ok = ct_test_support:verify_events(TestEvents, Events, Config). - -%%%----------------------------------------------------------------- -%%% - repeat_1(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -173,9 +154,6 @@ test_events(missing_conf) -> {?eh,stop_logging,[]} ]; -test_events(testspec_1) -> - []; - test_events(repeat_1) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec b/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec deleted file mode 100644 index 24d778ac44..0000000000 --- a/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec +++ /dev/null @@ -1,26 +0,0 @@ - -{config, "../cfgs/groups_2.1.cfg"}. -{alias, groups_2, "../groups_2"}. - -{suites, groups_2, groups_21_SUITE}. -%{skip_groups, groups_2, groups_21_SUITE, -% [test_group_1b, test_group_7], "Skip tg_1b & tg_7"}. -{skip_cases, groups_2, groups_21_SUITE, - [testcase_1b, testcase_3a], "Skip tc_1b & tc_3a"}. - -{groups, groups_2, groups_22_SUITE, - test_group_1a}. -{skip_cases, groups_2, groups_22_SUITE, - testcase_1a, "Skip tc_1a"}. - -{groups, groups_2, groups_22_SUITE, - test_group_1b}. -{skip_cases, groups_2, groups_22_SUITE, - testcase_1b, "Skip tc_1b"}. -%{skip_groups, groups_2, groups_21_SUITE, -% [test_group_3], "Skip tg_3"}. - -{groups, groups_2, groups_22_SUITE, - test_group_5}. -{skip_cases, groups_2, groups_22_SUITE, - [testcase_7a, testcase_7b], "Skip tc_7a & tc_7b"}. diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 6a75c8f789..899c0bc0c3 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -23,7 +23,7 @@ %%% -module(ct_test_support). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -include_lib("common_test/include/ct_event.hrl"). -export([init_per_suite/1, init_per_suite/2, end_per_suite/1, diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl new file mode 100644 index 0000000000..fb7f3fc9a9 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -0,0 +1,209 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_testspec_1_SUITE +%%% +%%% Description: +%%% Test test specifications +%%% +%%% The suites used for the test are located in the data directory. +%%%------------------------------------------------------------------- +-module(ct_testspec_1_SUITE). + +-compile(export_all). + +-include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +all(doc) -> + ["Run smoke tests of Common Test."]; + +all(suite) -> + [all_suites, skip_all_suites, + suite, skip_suite]. +% cases_1, skip_cases_1, +% groups_1, skip_groups_1]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%%%----------------------------------------------------------------- +%%% + +all_suites(Config) when is_list(Config) -> + Self = all_suites, + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,all}], + SpecFile = create_spec_file(?config(priv_dir, Config), + Self, TestSpec), + + {Opts,ERPid} = setup({spec,SpecFile}, Config), + ok = ct_test_support:run(Opts, Config), + ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Self, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(Self), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +skip_all_suites(Config) when is_list(Config) -> + Self = skip_all_suites, + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,all}, + {skip_suites,TestDir,all,"SKIPPED!"}], + SpecFile = create_spec_file(?config(priv_dir, Config), + Self, TestSpec), + + {Opts,ERPid} = setup({spec,SpecFile}, Config), + ok = ct_test_support:run(Opts, Config), + ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Self, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(Self), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% + +suite(Config) when is_list(Config) -> + Self = suite, + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,simple_1_SUITE}], + SpecFile = create_spec_file(?config(priv_dir, Config), + Self, TestSpec), + + {Opts,ERPid} = setup({spec,SpecFile}, Config), + ok = ct_test_support:run(Opts, Config), + ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Self, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(Self), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +skip_suite(Config) when is_list(Config) -> + Self = skip_suite, + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,[simple_1_SUITE,simple_2_SUITE]}, + {skip_suites,TestDir,simple_1_SUITE,"SKIPPED!"}], + SpecFile = create_spec_file(?config(priv_dir, Config), + Self, TestSpec), + + {Opts,ERPid} = setup({spec,SpecFile}, Config), + ok = ct_test_support:run(Opts, Config), + ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Self, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(Self), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + + +%%%----------------------------------------------------------------- +%%% + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +create_spec_file(SpecDir, TCName, TestSpec) -> + FileName = filename:join(SpecDir, + atom_to_list(TCName)++".spec"), + {ok,Dev} = file:open(FileName, [write]), + [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec], + file:close(Dev), + io:format("~nTest spec created here~n~n~s~n", + [FileName,FileName]), + FileName. + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). +%reformat(Events, _EH) -> +% Events. + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +%%%----------------------------------------------------------------- +events_to_check(Test) -> + %% 2 tests (ct:run_test + script_start) is default + events_to_check(Test, 2). + +events_to_check(_, 0) -> + []; +events_to_check(Test, N) -> + test_events(Test) ++ events_to_check(Test, N-1). + +test_events(_) -> + [ + ]. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl new file mode 100644 index 0000000000..4f11d8a0e8 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_11_SUITE.erl @@ -0,0 +1,281 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_11_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [testcase_1a,testcase_1b]}, + + {test_group_1b, [], [testcase_1a,testcase_1b]}, + + {test_group_2, [], [testcase_2a, + + {test_group_3, [], [testcase_3a, + testcase_3b]}, + testcase_2b]}, + + {test_group_4, [{test_group_5, [], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + {test_group_6, [{group, test_group_7}]}, + + {test_group_7, [testcase_7a,testcase_7b]} + ]. + +all() -> + [testcase_1, + {group, test_group_1a}, + {group, test_group_1b}, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1, + testcase_1a, testcase_1b, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + [{name,Group}] = ?config(tc_group_properties,Config), + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(Group), + init = ?config(suite,Config), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(Group), + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl new file mode 100644 index 0000000000..dbf1576c76 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl @@ -0,0 +1,314 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_12_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]}, + + {test_group_1b, [parallel], [testcase_1a,testcase_1b]}, + + {test_group_2, [parallel], [testcase_2a, + + {test_group_3, [{repeat,2}], + [testcase_3a, testcase_3b]}, + + testcase_2b]}, + + {test_group_4, [{test_group_5, [parallel], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [parallel], [{group, test_group_7}]}, + + {test_group_7, [sequence], [testcase_7a,testcase_7b]} + ]. + +all() -> + [{group, test_group_1a}, + {group, test_group_1b}, + testcase_1, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1a, testcase_1b, testcase_1c, + testcase_1, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + Cmt = + case {Group,?config(tc_group_properties,Config)} of + {test_group_1a,[{shuffle,S},{name,test_group_1a}]} -> + io_lib:format("shuffled, ~w", [S]); + {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; + {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; + {test_group_3,[{name,test_group_3},{repeat,2}]} -> "repeat 2"; + {test_group_3,[{name,test_group_3}]} -> "repeat 1"; + {test_group_4,[{name,test_group_4}]} -> ok; + {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; + {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; + {test_group_7,[{name,test_group_7},sequence]} -> "sequence" + end, + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + ct:comment(io_lib:format("~w, ~s", [Group,Cmt])), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_1c() -> + []. +testcase_1c(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1b,Config), + testcase_1c = ?config(testcase_1c,Config), + ok. + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + %% increase chance the done event will come + %% during execution of subgroup (could be + %% tricky to handle) + timer:sleep(3), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl new file mode 100644 index 0000000000..2533ac8e84 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_21_SUITE.erl @@ -0,0 +1,281 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_21_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [testcase_1a,testcase_1b]}, + + {test_group_1b, [], [testcase_1a,testcase_1b]}, + + {test_group_2, [], [testcase_2a, + + {test_group_3, [], [testcase_3a, + testcase_3b]}, + testcase_2b]}, + + {test_group_4, [{test_group_5, [], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [{group, test_group_7}]}, + + {test_group_7, [testcase_7a,testcase_7b]} + ]. + +all() -> + [testcase_1, + {group, test_group_1a}, + {group, test_group_1b}, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1, + testcase_1a, testcase_1b, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + [{name,Group}] = ?config(tc_group_properties,Config), + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + ct:comment(io_lib:format("~w", [Group])), + init = ?config(suite,Config), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl new file mode 100644 index 0000000000..cd517876df --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_2/groups_22_SUITE.erl @@ -0,0 +1,314 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_22_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%==================================================================== +%% COMMON TEST CALLBACK FUNCTIONS +%%==================================================================== + +suite() -> + [{timetrap,{minutes,1}}]. + +groups() -> + [ + {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]}, + + {test_group_1b, [parallel], [testcase_1a,testcase_1b]}, + + {test_group_2, [parallel], [testcase_2a, + + {test_group_3, [{repeat,1}], + [testcase_3a, testcase_3b]}, + + testcase_2b]}, + + {test_group_4, [{test_group_5, [parallel], [testcase_5a, + + {group, test_group_6}, + + testcase_5b]}]}, + + {test_group_6, [parallel], [{group, test_group_7}]}, + + {test_group_7, [sequence], [testcase_7a,testcase_7b]} + ]. + +all() -> + [{group, test_group_1a}, + {group, test_group_1b}, + testcase_1, + testcase_2, + {group, test_group_2}, + testcase_3, + {group, test_group_4}]. + +%% this func only for internal test purposes +grs_and_tcs() -> + {[ + test_group_1a, test_group_1b, + test_group_2, test_group_3, + test_group_4, test_group_5, + test_group_6, test_group_7 + ], + [ + testcase_1a, testcase_1b, testcase_1c, + testcase_1, + testcase_2, + testcase_2a, testcase_2b, + testcase_3a, testcase_3b, + testcase_3, + testcase_5a, testcase_5b, + testcase_7a, testcase_7b + ]}. + +%%-------------------------------------------------------------------- +%% Suite Configuration +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + [{suite,init}|Config]. + +end_per_suite(Config) -> + init = ?config(suite,Config). + +%%-------------------------------------------------------------------- +%% Group Configuration +%%-------------------------------------------------------------------- + +init_per_group(Group, Config) -> + Cmt = + case {Group,?config(tc_group_properties,Config)} of + {test_group_1a,[{shuffle,S},{name,test_group_1a}]} -> + io_lib:format("shuffled, ~w", [S]); + {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel"; + {test_group_2,[{name,test_group_2},parallel]} -> "parallel"; + {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1"; + {test_group_3,[{name,test_group_3}]} -> "repeat 0"; + {test_group_4,[{name,test_group_4}]} -> ok; + {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; + {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; + {test_group_7,[{name,test_group_7},sequence]} -> "sequence" + end, + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + ct:comment(io_lib:format("~w, ~s", [Group,Cmt])), + [{Group,Group} | Config]; + false -> + ct:fail({bad_group,Group}) + end. + +end_per_group(Group, Config) -> + {Grs,_} = grs_and_tcs(), + case lists:member(Group, Grs) of + true -> + init = ?config(suite,Config), + Group = ?config(Group,Config), + ok; + false -> + ct:fail({bad_group,Group}) + end. + +%%-------------------------------------------------------------------- +%% Testcase Configuration +%%-------------------------------------------------------------------- + +init_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + [{TestCase,TestCase} | Config]; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + +end_per_testcase(TestCase, Config) -> + {_,TCs} = grs_and_tcs(), + case lists:member(TestCase, TCs) of + true -> + init = ?config(suite,Config), + TestCase = ?config(TestCase,Config), + ok; + false -> + ct:fail({unknown_testcase,TestCase}) + end. + + +%%-------------------------------------------------------------------- +%% Testcases +%%-------------------------------------------------------------------- + +testcase_1a() -> + []. +testcase_1a(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + testcase_1a = ?config(testcase_1a,Config), + ok. +testcase_1b() -> + []. +testcase_1b(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1a,Config), + testcase_1b = ?config(testcase_1b,Config), + ok. + +testcase_1c() -> + []. +testcase_1c(Config) -> + init = ?config(suite,Config), + case ?config(test_group_1a,Config) of + test_group_1a -> ok; + _ -> + case ?config(test_group_1b,Config) of + test_group_1b -> ok; + _ -> ct:fail(no_group_data) + end + end, + undefined = ?config(testcase_1b,Config), + testcase_1c = ?config(testcase_1c,Config), + ok. + +testcase_1() -> + []. +testcase_1(Config) -> + init = ?config(suite,Config), + testcase_1 = ?config(testcase_1,Config), + ok. + +testcase_2() -> + []. +testcase_2(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_1a,Config), + undefined = ?config(test_group_1b,Config), + testcase_2 = ?config(testcase_2,Config), + ok. + +testcase_2a() -> + []. +testcase_2a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + testcase_2a = ?config(testcase_2a,Config), + ok. +testcase_2b() -> + []. +testcase_2b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + undefined = ?config(testcase_2a,Config), + testcase_2b = ?config(testcase_2b,Config), + ok. + +testcase_3a() -> + []. +testcase_3a(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_2b,Config), + testcase_3a = ?config(testcase_3a,Config), + ok. +testcase_3b() -> + []. +testcase_3b(Config) -> + init = ?config(suite,Config), + test_group_2 = ?config(test_group_2,Config), + test_group_3 = ?config(test_group_3,Config), + undefined = ?config(testcase_3a,Config), + testcase_3b = ?config(testcase_3b,Config), + ok. + +testcase_3() -> + []. +testcase_3(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_2,Config), + undefined = ?config(test_group_3,Config), + testcase_3 = ?config(testcase_3,Config), + ok. + +testcase_5a() -> + []. +testcase_5a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_3,Config), + testcase_5a = ?config(testcase_5a,Config), + %% increase chance the done event will come + %% during execution of subgroup (could be + %% tricky to handle) + timer:sleep(3), + ok. +testcase_5b() -> + []. +testcase_5b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + undefined = ?config(testcase_5a,Config), + testcase_5b = ?config(testcase_5b,Config), + ok. + +testcase_7a() -> + []. +testcase_7a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + testcase_7a = ?config(testcase_7a,Config), + ok. +testcase_7b() -> + []. +testcase_7b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_7 = ?config(test_group_7,Config), + undefined = ?config(testcase_7a,Config), + testcase_7b = ?config(testcase_7b,Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl new file mode 100644 index 0000000000..b789851134 --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_1_SUITE.erl @@ -0,0 +1,146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(simple_1_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [ + {timetrap,{seconds,10}} + ]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + [{ips,ips_data} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ips_data = ?config(ips, Config). + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(TestCase, Config) -> + [{TestCase,{TestCase,data}} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(TestCase, Config) -> + {TestCase,data} = ?config(TestCase, Config). + +%%-------------------------------------------------------------------- +%% Function: all() -> TestCases | {skip,Reason} +%% +%% TestCases = [TestCase | {sequence,SeqName}] +%% TestCase = atom() +%% Name of a test case. +%% SeqName = atom() +%% Name of a test case sequence. +%% Reason = term() +%% The reason for skipping all test cases. +%% +%% Description: Returns the list of test cases that are to be executed. +%%-------------------------------------------------------------------- +all() -> + [tc1, + tc2]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +tc1() -> + [{userdata,{info, "This is a testcase"}}]. + +tc1(Config) -> + ips_data = ?config(ips, Config), + {tc1,data} = ?config(tc1, Config), + ok. + +tc2() -> + [{timetrap,5000}]. + +tc2(Config) -> + ips_data = ?config(ips, Config), + undefined = ?config(tc1, Config), + {tc2,data} = ?config(tc2, Config), + ok. diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl new file mode 100644 index 0000000000..eb7e9cdf7b --- /dev/null +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/suites_1/simple_2_SUITE.erl @@ -0,0 +1,146 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(simple_2_SUITE). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +%%-------------------------------------------------------------------- +%% COMMON TEST CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% +%% Info = [tuple()] +%% List of key/value pairs. +%% +%% Description: Returns list of tuples to set default properties +%% for the suite. +%% +%% Note: The suite/0 function is only meant to be used to return +%% default data values, not perform any other operations. +%%-------------------------------------------------------------------- +suite() -> + [ + {timetrap,{seconds,10}} + ]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the suite. +%% +%% Description: Initialization before the suite. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + [{ips,ips_data} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after the suite. +%%-------------------------------------------------------------------- +end_per_suite(Config) -> + ips_data = ?config(ips, Config). + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is about to run. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% Reason = term() +%% The reason for skipping the test case. +%% +%% Description: Initialization before each test case. +%% +%% Note: This function is free to add any key/value pairs to the Config +%% variable, but should NOT alter/remove any existing entries. +%%-------------------------------------------------------------------- +init_per_testcase(TestCase, Config) -> + [{TestCase,{TestCase,data}} | Config]. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% +%% TestCase = atom() +%% Name of the test case that is finished. +%% Config0 = Config1 = [tuple()] +%% A list of key/value pairs, holding the test case configuration. +%% +%% Description: Cleanup after each test case. +%%-------------------------------------------------------------------- +end_per_testcase(TestCase, Config) -> + {TestCase,data} = ?config(TestCase, Config). + +%%-------------------------------------------------------------------- +%% Function: all() -> TestCases | {skip,Reason} +%% +%% TestCases = [TestCase | {sequence,SeqName}] +%% TestCase = atom() +%% Name of a test case. +%% SeqName = atom() +%% Name of a test case sequence. +%% Reason = term() +%% The reason for skipping all test cases. +%% +%% Description: Returns the list of test cases that are to be executed. +%%-------------------------------------------------------------------- +all() -> + [tc1, + tc2]. + + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +tc1() -> + [{userdata,{info, "This is a testcase"}}]. + +tc1(Config) -> + ips_data = ?config(ips, Config), + {tc1,data} = ?config(tc1, Config), + ok. + +tc2() -> + [{timetrap,5000}]. + +tc2(Config) -> + ips_data = ?config(ips, Config), + undefined = ?config(tc1, Config), + {tc2,data} = ?config(tc2, Config), + ok. diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index d1f53f3c39..a28a73a8ed 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1564,8 +1564,8 @@ temp_nodename([Chr|Base], Acc) -> %% of cases can not be calculated and NoOfCases = unknown. count_test_cases(TopCases, SkipCases) when is_list(TopCases) -> case collect_all_cases(TopCases, SkipCases) of - {error,_Why} -> - error; + {error,_Why} = Error -> + Error; TestSpec -> {get_suites(TestSpec, []), case remove_conf(TestSpec) of @@ -2067,10 +2067,6 @@ run_test_cases(TestSpec, Config, TimetrapData) -> html_convert_modules(TestSpec, Config), - %%! For readable tracing... - %%! Config1 = [{data_dir,""},{priv_dir,""},{nodes,[]}], - %%! run_test_cases_loop(TestSpec, [[]], TimetrapData, [], []), - run_test_cases_loop(TestSpec, [Config], TimetrapData, [], []), maybe_get_privdir(), @@ -3552,7 +3548,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Where, case test_server_sup:framework_call(warn, [nodes], true) of true -> case catch controller_call(kill_slavenodes) of - {'EXIT',_}=Exit -> + {'EXIT',_} = Exit -> print(minor, "WARNING: There might be slavenodes left in the" " system. I tried to kill them, but I failed: ~p\n", @@ -4444,9 +4440,9 @@ collect_cases([Case|Cs0], St0) -> case collect_cases(Cs0, St1) of {ok,FlatCases2,St} -> {ok,FlatCases1 ++ FlatCases2,St}; - {error,_Reason}=Error -> Error + {error,_Reason} = Error -> Error end; - {error,_Reason}=Error -> Error + {error,_Reason} = Error -> Error end; @@ -4482,39 +4478,55 @@ collect_cases({conf,Props,InitMF,CaseList,FinF}, St) when is_atom(FinF) -> Props1 -> collect_cases({conf,Props1,InitMF,CaseList,{St#cc.mod,FinF}}, St) end; -collect_cases({conf,Props,InitMF,CaseList,FinMF}, St0) -> - case collect_cases(CaseList, St0) of - {ok,[],_St}=Empty -> - Empty; - {ok,FlatCases,St} -> +collect_cases({conf,Props,InitMF,CaseList,FinMF} = Conf, St) -> + case init_props(Props) of + {error,_} -> + {ok,[],St}; + Props1 -> Ref = make_ref(), - case in_skip_list(InitMF, St#cc.skip) of - {true,Comment} -> + Skips = St#cc.skip, + case in_skip_list({St#cc.mod,Conf}, Skips) of + {true,Comment} -> % conf init skipped {ok,[{skip_case,{conf,Ref,InitMF,Comment}} | - FlatCases ++ [{conf,Ref,[],FinMF}]],St}; + [] ++ [{conf,Ref,[],FinMF}]],St}; + {true,Name,Comment} when is_atom(Name) -> % all cases skipped + {ok,[{skip_case,{{St#cc.mod,{group,Name}},Comment}}],St}; + {true,ToSkip,_} when is_list(ToSkip) -> % some cases skipped + case collect_cases(CaseList, + St#cc{skip=ToSkip++Skips}) of + {ok,[],_St} = Empty -> + Empty; + {ok,FlatCases,St1} -> + {ok,[{conf,Ref,Props1,InitMF} | + FlatCases ++ [{conf,Ref, + keep_name(Props1), + FinMF}]],St1#cc{skip=Skips}}; + {error,_Reason} = Error -> + Error + end; false -> - case init_props(Props) of - {error,_} -> - {ok,[],St}; - Props1 -> + case collect_cases(CaseList, St) of + {ok,[],_St} = Empty -> + Empty; + {ok,FlatCases,St1} -> {ok,[{conf,Ref,Props1,InitMF} | FlatCases ++ [{conf,Ref, keep_name(Props1), - FinMF}]],St} + FinMF}]],St1}; + {error,_Reason} = Error -> + Error end - end; - {error,_Reason}=Error -> - Error + end end; collect_cases({make,InitMFA,CaseList,FinMFA}, St0) -> case collect_cases(CaseList, St0) of - {ok,[],_St}=Empty -> Empty; + {ok,[],_St} = Empty -> Empty; {ok,FlatCases,St} -> Ref = make_ref(), {ok,[{make,Ref,InitMFA}|FlatCases ++ [{make,Ref,FinMFA}]],St}; - {error,_Reason}=Error -> Error + {error,_Reason} = Error -> Error end; collect_cases({Module, Cases}, St) when is_list(Cases) -> @@ -4535,6 +4547,9 @@ collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) -> collect_cases(Other, St) -> {error,{bad_subtest_spec,St#cc.mod,Other}}. +collect_case({_Mod,{conf,_,_,_,_}=Conf}, St) -> + collect_cases(Conf, St); + collect_case(MFA, St) -> case in_skip_list(MFA, St#cc.skip) of {true,Comment} -> @@ -4580,6 +4595,8 @@ collect_subcases(Mod, Case, MFA, St, Suite) -> %%%! --- END Kept for backwards compatibilty --- {Skip,Reason} when Skip==skip; Skip==skipped -> {ok,[{skip_case,{MFA,Reason}}],St}; + {error,Reason} -> + throw(Reason); SubCases -> collect_case_subcases(Mod, Case, SubCases, St) end. @@ -4641,6 +4658,45 @@ check_deny_req(Req, DenyList) -> false -> granted end. +in_skip_list({Mod,{conf,Props,InitMF,_CaseList,_FinMF}}, SkipList) -> + case in_skip_list(InitMF, SkipList) of + {true,_} = Yes -> + Yes; + _ -> + case proplists:get_value(name, Props) of + undefined -> + false; + Name -> + ToSkip = + lists:flatmap( + fun({M,{conf,SProps,_,SCaseList,_},Cmt}) when + M == Mod -> + case proplists:get_value(name, SProps) of + Name -> + case SCaseList of + all -> + [{M,all,Cmt}]; + _ -> + [{M,F,Cmt} || F <- SCaseList] + end; + _ -> + [] + end; + (_) -> + [] + end, SkipList), + case ToSkip of + [] -> + false; + _ -> + case lists:keysearch(all, 2, ToSkip) of + {value,{_,_,Cmt}} -> {true,Name,Cmt}; + _ -> {true,ToSkip,""} + end + end + end + end; + in_skip_list({Mod,Func,_Args}, SkipList) -> in_skip_list({Mod,Func}, SkipList); in_skip_list({Mod,Func}, [{Mod,Funcs,Comment}|SkipList]) when is_list(Funcs) -> -- cgit v1.2.3 From 31b52954dc667861a2e7cd4edba100132499cd5f Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 29 Jun 2010 17:13:05 +0200 Subject: Add new option to label test runs With the option 'label', the test run gets a user defined name that Common Test prints in the overview log files. --- lib/common_test/src/ct_logs.erl | 66 ++++-- lib/common_test/src/ct_run.erl | 63 ++++-- lib/common_test/src/ct_testspec.erl | 41 +++- lib/common_test/src/ct_util.hrl | 1 + lib/common_test/test/ct_testspec_1_SUITE.erl | 312 +++++++++++++++++++++------ 5 files changed, 390 insertions(+), 93 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 5683d06aa7..3ed0ea5c40 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -716,7 +716,7 @@ make_last_run_index1(StartTime,IndexName) -> [Log]; Logs -> case read_totals_file(?totals_name) of - {_Node,Logs0,_Totals} -> + {_Node,_Lbl,Logs0,_Totals} -> insert_dirs(Logs,Logs0); _ -> %% someone deleted the totals file!? @@ -728,10 +728,15 @@ make_last_run_index1(StartTime,IndexName) -> {ok,Bin} -> binary_to_term(Bin); _ -> [] end, - {ok,Index0,Totals} = make_last_run_index(Logs1, index_header(StartTime), + Label = case application:get_env(common_test, test_label) of + {ok,Lbl} -> Lbl; + _ -> undefined + end, + {ok,Index0,Totals} = make_last_run_index(Logs1, + index_header(Label,StartTime), 0, 0, 0, 0, 0, Missing), %% write current Totals to file, later to be used in all_runs log - write_totals_file(?totals_name,Logs1,Totals), + write_totals_file(?totals_name,Label,Logs1,Totals), Index = [Index0|index_footer()], case force_write_file(IndexName, Index) of ok -> @@ -937,8 +942,16 @@ term_to_text(Term) -> %%% Headers and footers. -index_header(StartTime) -> - [header("Test Results " ++ format_time(StartTime)) | +index_header(Label, StartTime) -> + Head = + case Label of + undefined -> + header("Test Results", format_time(StartTime)); + _ -> + header("Test Results for \"" ++ Label ++ "\"", + format_time(StartTime)) + end, + [Head | ["
\n", "

Common Test Framework Log

", " "BGCOLOR=\"",?table_color1,"\">\n" "\n" "\n" + "\n" "\n" "\n" "\n" @@ -987,12 +1001,23 @@ all_runs_header() -> "\n"]]. header(Title) -> + header1(Title, ""). +header(Title, SubTitle) -> + header1(Title, SubTitle). + +header1(Title, SubTitle) -> + SubTitleHTML = if SubTitle =/= "" -> + ["
\n", + "

" ++ SubTitle ++ "

\n", + "
\n
\n"]; + true -> "
\n" + end, ["\n" "\n" "\n", "\n", - "" ++ Title ++ "\n", + "" ++ Title ++ " " ++ SubTitle ++ "\n", "\n", "\n", @@ -1004,6 +1029,7 @@ header(Title) -> "
\n", "

" ++ Title ++ "

\n", "
\n", + SubTitleHTML, "\n"]. @@ -1217,7 +1243,7 @@ runentry(Dir, BasicHtml) -> TotalsFile = filename:join(Dir,?totals_name), TotalsStr = case read_totals_file(TotalsFile) of - {Node,Logs,{TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}} -> + {Node,Label,Logs,{TotSucc,TotFail,UserSkip,AutoSkip,NotBuilt}} -> TotFailStr = if TotFail > 0 -> ["", @@ -1263,6 +1289,7 @@ runentry(Dir, BasicHtml) -> end, Total = TotSucc+TotFail+AllSkip, A = ["\n", + "\n", "\n"], B = if BasicHtml -> ["\n"]; @@ -1286,14 +1313,16 @@ runentry(Dir, BasicHtml) -> "\n" "\n"]. -write_totals_file(Name,Logs,Totals) -> +write_totals_file(Name,Label,Logs,Totals) -> AbsName = ?abs(Name), notify_and_lock_file(AbsName), force_write_file(AbsName, term_to_binary({atom_to_list(node()), - Logs,Totals})), + Label,Logs,Totals})), notify_and_unlock_file(AbsName). +%% this function needs to convert from old formats to new so that old +%% test results (prev ct versions) can be listed together with new read_totals_file(Name) -> AbsName = ?abs(Name), notify_and_lock_file(AbsName), @@ -1303,12 +1332,23 @@ read_totals_file(Name) -> case catch binary_to_term(Bin) of {'EXIT',_Reason} -> % corrupt file {"-",[],undefined}; - R = {Node,Ls,Tot} -> + {Node,Label,Ls,Tot} -> % all info available + Label1 = case Label of + undefined -> "-"; + _ -> Label + end, + case Tot of + {_Ok,_Fail,_USkip,_ASkip,_NoBuild} -> % latest format + {Node,Label1,Ls,Tot}; + {TotSucc,TotFail,AllSkip,NotBuilt} -> + {Node,Label1,Ls,{TotSucc,TotFail,AllSkip,undefined,NotBuilt}} + end; + {Node,Ls,Tot} -> % no label found case Tot of - {_,_,_,_,_} -> % latest format - R; + {_Ok,_Fail,_USkip,_ASkip,_NoBuild} -> % latest format + {Node,"-",Ls,Tot}; {TotSucc,TotFail,AllSkip,NotBuilt} -> - {Node,Ls,{TotSucc,TotFail,AllSkip,undefined,NotBuilt}} + {Node,"-",Ls,{TotSucc,TotFail,AllSkip,undefined,NotBuilt}} end; %% for backwards compatibility {Ls,Tot} -> {"-",Ls,Tot}; diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index adb49aa44d..0b4a508321 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -45,7 +45,8 @@ -define(abs(Name), filename:absname(Name)). -define(testdir(Name, Suite), ct_util:get_testdir(Name, Suite)). --record(opts, {vts, +-record(opts, {label, + vts, shell, cover, coverspec, @@ -158,6 +159,7 @@ script_start(Args) -> script_start1(Parent, Args) -> %% read general start flags + Label = get_start_opt(label, fun([Lbl]) -> Lbl end, Args), Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), @@ -230,7 +232,7 @@ script_start1(Parent, Args) -> application:set_env(common_test, basic_html, true) end, - StartOpts = #opts{vts = Vts, shell = Shell, cover = Cover, + StartOpts = #opts{label = Label, vts = Vts, shell = Shell, cover = Cover, logdir = LogDir, event_handlers = EvHandlers, include = IncludeDirs, silent_connections = SilentConns, @@ -289,6 +291,9 @@ script_start2(StartOpts = #opts{vts = undefined, TS -> SpecStartOpts = get_data_for_node(TS, node()), + Label = choose_val(StartOpts#opts.label, + SpecStartOpts#opts.label), + LogDir = choose_val(StartOpts#opts.logdir, SpecStartOpts#opts.logdir), @@ -304,7 +309,8 @@ script_start2(StartOpts = #opts{vts = undefined, SpecStartOpts#opts.include]), application:set_env(common_test, include, AllInclude), - {TS,StartOpts#opts{testspecs = Specs, + {TS,StartOpts#opts{label = Label, + testspecs = Specs, cover = Cover, logdir = LogDir, config = SpecStartOpts#opts.config, @@ -430,8 +436,12 @@ script_start4(#opts{vts = true, config = Config, event_handlers = EvHandlers, end, [], Config), vts:init_data(ConfigFiles, EvHandlers, ?abs(LogDir), Tests); -script_start4(#opts{shell = true, config = Config, event_handlers = EvHandlers, +script_start4(#opts{label = Label, shell = true, config = Config, + event_handlers = EvHandlers, logdir = LogDir, testspecs = Specs}, _Args) -> + %% label - used by ct_logs + application:set_env(common_test, test_label, Label), + InstallOpts = [{config,Config},{event_handler,EvHandlers}], if Config == [] -> ok; @@ -616,6 +626,10 @@ run_test(StartOpts) when is_list(StartOpts) -> end. run_test1(StartOpts) -> + %% label + Label = get_start_opt(label, fun(Lbl) when is_list(Lbl) -> Lbl; + (Lbl) when is_atom(Lbl) -> atom_to_list(Lbl) + end, StartOpts), %% logdir LogDir = get_start_opt(logdir, fun(LD) when is_list(LD) -> LD end, StartOpts), @@ -714,7 +728,8 @@ run_test1(StartOpts) -> %% stepped execution Step = get_start_opt(step, value, StartOpts), - Opts = #opts{cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, + Opts = #opts{label = Label, + cover = Cover, step = Step, logdir = LogDir, config = CfgFiles, event_handlers = EvHandlers, include = Include, silent_connections = SilentConns, stylesheet = Stylesheet, @@ -750,6 +765,8 @@ run_spec_file(Relaxed, exit(CTReason); TS -> SpecOpts = get_data_for_node(TS, node()), + Label = choose_val(Opts#opts.label, + SpecOpts#opts.label), LogDir = choose_val(Opts#opts.logdir, SpecOpts#opts.logdir), AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), @@ -769,7 +786,8 @@ run_spec_file(Relaxed, which(logdir,LogDir), AllEvHs) of ok -> - Opts1 = Opts#opts{cover = Cover, + Opts1 = Opts#opts{label = Label, + cover = Cover, logdir = which(logdir, LogDir), config = AllConfig, event_handlers = AllEvHs, @@ -954,14 +972,16 @@ run_testspec1(TestSpec) -> end end. -get_data_for_node(#testspec{logdir=LogDirs, - cover=CoverFs, - config=Cfgs, - userconfig=UsrCfgs, - event_handler=EvHs, - include=Incl, - multiply_timetraps=MTs, - scale_timetraps=STs}, Node) -> +get_data_for_node(#testspec{label = Labels, + logdir = LogDirs, + cover = CoverFs, + config = Cfgs, + userconfig = UsrCfgs, + event_handler = EvHs, + include = Incl, + multiply_timetraps = MTs, + scale_timetraps = STs}, Node) -> + Label = proplists:get_value(Node, Labels), LogDir = case proplists:get_value(Node, LogDirs) of undefined -> "."; Dir -> Dir @@ -973,7 +993,8 @@ get_data_for_node(#testspec{logdir=LogDirs, [CBF || {N,CBF} <- UsrCfgs, N==Node], EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node], Include = [I || {N,I} <- Incl, N==Node], - #opts{logdir = LogDir, + #opts{label = Label, + logdir = LogDir, cover = Cover, config = ConfigFiles, event_handlers = EvHandlers, @@ -1112,7 +1133,17 @@ do_run(Tests, Misc, LogDir) when is_list(Misc) -> do_run(Tests, [], Opts1#opts{logdir = LogDir}, []). do_run(Tests, Skip, Opts, Args) -> - #opts{cover = Cover} = Opts, + #opts{label = Label, cover = Cover} = Opts, + + %% label - used by ct_logs + TestLabel = + if Label == undefined -> undefined; + is_atom(Label) -> atom_to_list(Label); + is_list(Label) -> Label; + true -> undefined + end, + application:set_env(common_test, test_label, TestLabel), + case code:which(test_server) of non_existing -> exit({error,no_path_to_test_server}); diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 9d2a791406..1aa3a859ab 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -440,6 +440,15 @@ save_nodes(Nodes,Spec=#testspec{nodes=NodeRefs}) -> list_nodes(#testspec{nodes=NodeRefs}) -> lists:map(fun({_Ref,Node}) -> Node end, NodeRefs). + + +%% --------------------------------------------------------- +%% / \ +%% | When adding tests, remember to update valid_terms/0 also! | +%% \ / +%% --------------------------------------------------------- + + %% Associate a "global" logdir with all nodes %% except those with specific logdir, e.g: %% ["/tmp/logdir",{ct1@finwe,"/tmp/logdir2"}] @@ -465,6 +474,24 @@ add_tests([{logdir,Node,Dir}|Ts],Spec) -> add_tests([{logdir,Dir}|Ts],Spec) -> add_tests([{logdir,all_nodes,Dir}|Ts],Spec); +%% --- label --- +add_tests([{label,all_nodes,Lbl}|Ts],Spec) -> + Labels = Spec#testspec.label, + Tests = [{label,N,Lbl} || N <- list_nodes(Spec), + lists:keymember(ref2node(N,Spec#testspec.nodes), + 1,Labels) == false], + add_tests(Tests++Ts,Spec); +add_tests([{label,Nodes,Lbl}|Ts],Spec) when is_list(Nodes) -> + Ts1 = separate(Nodes,label,[Lbl],Ts,Spec#testspec.nodes), + add_tests(Ts1,Spec); +add_tests([{label,Node,Lbl}|Ts],Spec) -> + Labels = Spec#testspec.label, + Labels1 = [{ref2node(Node,Spec#testspec.nodes),Lbl} | + lists:keydelete(ref2node(Node,Spec#testspec.nodes),1,Labels)], + add_tests(Ts,Spec#testspec{label=Labels1}); +add_tests([{label,Lbl}|Ts],Spec) -> + add_tests([{label,all_nodes,Lbl}|Ts],Spec); + %% --- cover --- add_tests([{cover,all_nodes,File}|Ts],Spec) -> Tests = lists:map(fun(N) -> {cover,N,File} end, list_nodes(Spec)), @@ -1013,22 +1040,32 @@ valid_terms() -> {cover,3}, {config,2}, {config,3}, - {userconfig, 2}, - {userconfig, 3}, + {userconfig,2}, + {userconfig,3}, {alias,3}, {logdir,2}, {logdir,3}, {event_handler,2}, {event_handler,3}, {event_handler,4}, + {multiply_timetraps,2}, + {multiply_timetraps,3}, + {scale_timetraps,2}, + {scale_timetraps,3}, {include,2}, {include,3}, {suites,3}, {suites,4}, + {groups,4}, + {groups,5}, + {groups,6}, {cases,4}, {cases,5}, {skip_suites,4}, {skip_suites,5}, + {skip_groups,5}, + {skip_groups,6}, + {skip_groups,7}, {skip_cases,5}, {skip_cases,6} ]. diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 54eed29415..ee973f6220 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -30,6 +30,7 @@ -record(testspec, {spec_dir, nodes=[], init=[], + label=[], logdir=["."], cover=[], config=[], diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index fb7f3fc9a9..c3840d0425 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -61,9 +61,18 @@ all(doc) -> all(suite) -> [all_suites, skip_all_suites, - suite, skip_suite]. -% cases_1, skip_cases_1, -% groups_1, skip_groups_1]. + suite, skip_suite, + all_testcases, skip_all_testcases, + testcase, skip_testcase, + all_groups, skip_all_groups, + group, skip_group, + group_all_testcases, skip_group_all_testcases, + group_testcase, skip_group_testcase, + topgroup, + subgroup, skip_subgroup, + subgroup_all_testcases, skip_subgroup_all_testcases, + subgroup_testcase, skip_subgroup_testcase, + only_skip]. %%-------------------------------------------------------------------- %% TEST CASES @@ -73,102 +82,279 @@ all(suite) -> %%% all_suites(Config) when is_list(Config) -> - Self = all_suites, DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "suites_1"), - TestSpec = [{suites,TestDir,all}], - SpecFile = create_spec_file(?config(priv_dir, Config), - Self, TestSpec), - - {Opts,ERPid} = setup({spec,SpecFile}, Config), - ok = ct_test_support:run(Opts, Config), - ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), - Events = ct_test_support:get_events(ERPid, Config), + TestSpec = [{label,"all_suites"}, + {suites,TestDir,all}], - ct_test_support:log_events(Self, - reformat(Events, ?eh), - ?config(priv_dir, Config)), - - TestEvents = events_to_check(Self), - ok = ct_test_support:verify_events(TestEvents, Events, Config). + setup_and_execute(all_suites, TestSpec, Config). skip_all_suites(Config) when is_list(Config) -> - Self = skip_all_suites, DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "suites_1"), - TestSpec = [{suites,TestDir,all}, + TestSpec = [{label,skip_all_suites}, + {suites,TestDir,all}, {skip_suites,TestDir,all,"SKIPPED!"}], - SpecFile = create_spec_file(?config(priv_dir, Config), - Self, TestSpec), - {Opts,ERPid} = setup({spec,SpecFile}, Config), - ok = ct_test_support:run(Opts, Config), - ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), - Events = ct_test_support:get_events(ERPid, Config), - - ct_test_support:log_events(Self, - reformat(Events, ?eh), - ?config(priv_dir, Config)), - - TestEvents = events_to_check(Self), - ok = ct_test_support:verify_events(TestEvents, Events, Config). + setup_and_execute(skip_all_suites, TestSpec, Config). %%%----------------------------------------------------------------- %%% suite(Config) when is_list(Config) -> - Self = suite, DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "suites_1"), - TestSpec = [{suites,TestDir,simple_1_SUITE}], - SpecFile = create_spec_file(?config(priv_dir, Config), - Self, TestSpec), + TestSpec = [{label,undefined}, + {suites,TestDir,simple_1_SUITE}], - {Opts,ERPid} = setup({spec,SpecFile}, Config), - ok = ct_test_support:run(Opts, Config), - ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), - Events = ct_test_support:get_events(ERPid, Config), - - ct_test_support:log_events(Self, - reformat(Events, ?eh), - ?config(priv_dir, Config)), - - TestEvents = events_to_check(Self), - ok = ct_test_support:verify_events(TestEvents, Events, Config). + setup_and_execute(suite, TestSpec, Config). skip_suite(Config) when is_list(Config) -> - Self = skip_suite, DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "suites_1"), TestSpec = [{suites,TestDir,[simple_1_SUITE,simple_2_SUITE]}, {skip_suites,TestDir,simple_1_SUITE,"SKIPPED!"}], - SpecFile = create_spec_file(?config(priv_dir, Config), - Self, TestSpec), - {Opts,ERPid} = setup({spec,SpecFile}, Config), - ok = ct_test_support:run(Opts, Config), - ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), - Events = ct_test_support:get_events(ERPid, Config), + setup_and_execute(skip_suite, TestSpec, Config). - ct_test_support:log_events(Self, - reformat(Events, ?eh), - ?config(priv_dir, Config)), +%%%----------------------------------------------------------------- +%%% - TestEvents = events_to_check(Self), - ok = ct_test_support:verify_events(TestEvents, Events, Config). +all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{cases,TestDir,simple_1_SUITE,all}], + + setup_and_execute(all_testcases, TestSpec, Config). + +skip_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{suites,TestDir,[simple_1_SUITE]}, + {skip_cases,TestDir,simple_1_SUITE,all,"SKIPPED!"}], + + setup_and_execute(skip_all_testcases, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{cases,TestDir,simple_1_SUITE,tc1}], + + setup_and_execute(testcase, TestSpec, Config). + +skip_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "suites_1"), + TestSpec = [{cases,TestDir,simple_1_SUITE,[tc1,tc2]}, + {cases,TestDir,simple_2_SUITE,[tc2,tc1]}, + {skip_cases,TestDir,simple_1_SUITE,[tc1],"SKIPPED!"}, + {skip_cases,TestDir,simple_2_SUITE,tc2,"SKIPPED!"}], + + setup_and_execute(skip_testcase, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +all_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,all}], + + setup_and_execute(all_groups, TestSpec, Config). + +skip_all_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,all}, + {skip_groups,TestDir,groups_11_SUITE,all,"SKIPPED!"}], + + setup_and_execute(skip_all_groups, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a}], + + setup_and_execute(group, TestSpec, Config). + +skip_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a, + test_group_1b]}, + {skip_groups,TestDir,groups_11_SUITE, + [test_group_1b,test_group_2],"SKIPPED!"}], + + setup_and_execute(skip_group, TestSpec, Config). + + +%%%----------------------------------------------------------------- +%%% + +group_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,{cases,all}}], + + setup_and_execute(group_all_testcases, TestSpec, Config). + +skip_group_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a, + test_group_1b]}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1b,{cases,all},"SKIPPED!"}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1a,{cases,all},"SKIPPED!"}], + + setup_and_execute(skip_group_all_testcases, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +group_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,{cases,testcase_1a}}], + + setup_and_execute(group_testcase, TestSpec, Config). + +skip_group_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a, + {cases,[testcase_1a,testcase_1b]}}, + {groups,TestDir,groups_11_SUITE,test_group_1b, + {cases,[testcase_1a,testcase_1b]}}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1a,{cases,testcase_1b},"SKIPPED!"}, + {skip_groups,TestDir,groups_11_SUITE, + test_group_1b,{cases,[testcase_1a]},"SKIPPED!"}], + + setup_and_execute(skip_group_testcase, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% +topgroup(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}], + + setup_and_execute(topgroup, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +subgroup(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_3}], + + setup_and_execute(subgroup, TestSpec, Config). + +skip_subgroup(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]}, + {skip_groups,TestDir,groups_12_SUITE, + test_group_3,"SKIPPED!"}], + + setup_and_execute(skip_subgroup, TestSpec, Config). %%%----------------------------------------------------------------- %%% +subgroup_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE, + test_group_5,{cases,all}}], + + setup_and_execute(subgroup_all_testcases, TestSpec, Config). + +skip_subgroup_all_testcases(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_4}, + {skip_groups,TestDir,groups_12_SUITE, + test_group_5,{cases,all},"SKIPPED!"}], + + setup_and_execute(skip_subgroup_all_testcases, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + +subgroup_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE, + test_group_7,{cases,testcase_7a}}], + + setup_and_execute(subgroup_testcase, TestSpec, Config). + +skip_subgroup_testcase(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5}, + {skip_groups,TestDir,groups_12_SUITE, + test_group_7,{cases,[testcase_7a,testcase_7b]},"SKIPPED!"}], + + setup_and_execute(skip_subgroup_testcase, TestSpec, Config). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- +setup_and_execute(TCName, TestSpec, Config) -> + SpecFile = create_spec_file(?config(priv_dir, Config), + TCName, TestSpec), + TestTerms = + case lists:keymember(label, 1, TestSpec) of + true -> [{spec,SpecFile}]; + false -> [{spec,SpecFile},{label,TCName}] + end, + {Opts,ERPid} = setup(TestTerms, Config), + ok = ct_test_support:run(Opts, Config), + ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(TCName, + reformat(Events, ?eh), + ?config(priv_dir, Config)), + + TestEvents = events_to_check(TCName), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + create_spec_file(SpecDir, TCName, TestSpec) -> FileName = filename:join(SpecDir, atom_to_list(TCName)++".spec"), @@ -179,11 +365,13 @@ create_spec_file(SpecDir, TCName, TestSpec) -> [FileName,FileName]), FileName. -setup(Test, Config) -> +setup(Test, Config) when is_tuple(Test) -> + setup([Test], Config); +setup(Tests, Config) -> Opts0 = ct_test_support:get_opts(Config), Level = ?config(trace_level, Config), EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], - Opts = Opts0 ++ [Test,{event_handler,{?eh,EvHArgs}}], + Opts = Opts0 ++ Tests ++ [{event_handler,{?eh,EvHArgs}}], ERPid = ct_test_support:start_event_receiver(Config), {Opts,ERPid}. -- cgit v1.2.3 From 79851b599df5caf5e2101d834a843e3837c6a6ae Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 29 Jun 2010 18:03:09 +0200 Subject: Improve handling of test case group specifications --- lib/common_test/src/ct_framework.erl | 28 ++++++++++++++- lib/common_test/src/ct_logs.erl | 51 +++++++++++++++++++--------- lib/common_test/src/ct_run.erl | 13 ++++--- lib/common_test/src/ct_testspec.erl | 2 +- lib/common_test/test/ct_test_support.erl | 5 +-- lib/common_test/test/ct_testspec_1_SUITE.erl | 32 +++++++++++++++-- 6 files changed, 104 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 25915b9db2..1c8a14574f 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -29,7 +29,7 @@ -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). --export([make_conf/5]). +-export([make_all_conf/3, make_conf/5]). -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -850,6 +850,32 @@ expand(Mod, Name, Defs) -> throw({error,list_to_atom(E)}) end. +make_all_conf(Dir, Mod, _Props) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_all_conf(Mod). + +make_all_conf(Mod) -> + case catch apply(Mod, groups, []) of + {'EXIT',_} -> + {error,{invalid_group_definition,Mod}}; + GroupDefs when is_list(GroupDefs) -> + case catch find_groups(Mod, all, all, GroupDefs) of + {error,_} = Error -> + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Error properly + [{?MODULE,error_in_suite,[[Error]]}]; + [] -> + {error,{invalid_group_spec,Mod}}; + ConfTests -> + ConfTests + end + end. + make_conf(Dir, Mod, Name, Props, TestSpec) -> case code:is_loaded(Mod) of false -> diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 3ed0ea5c40..bb859d3b72 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -57,7 +57,7 @@ -define(table_color2,"#E4F0FE"). -define(table_color3,"#F0F8FF"). --define(testname_width, 70). +-define(testname_width, 55). -define(abs(Name), filename:absname(Name)). @@ -766,7 +766,7 @@ make_last_run_index([Name|Rest], Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, Missing); LastLogDir -> SuiteName = filename:rootname(filename:basename(Name)), - case make_one_index_entry(SuiteName, LastLogDir, false, Missing) of + case make_one_index_entry(SuiteName, LastLogDir, "-", false, Missing) of {Result1,Succ,Fail,USkip,ASkip,NotBuilt} -> %% for backwards compatibility AutoSkip1 = case catch AutoSkip+ASkip of @@ -785,18 +785,18 @@ make_last_run_index([], Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuil {ok, [Result|total_row(TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, false)], {TotSucc,TotFail,UserSkip,AutoSkip,TotNotBuilt}}. -make_one_index_entry(SuiteName, LogDir, All, Missing) -> +make_one_index_entry(SuiteName, LogDir, Label, All, Missing) -> case count_cases(LogDir) of {Succ,Fail,UserSkip,AutoSkip} -> NotBuilt = not_built(SuiteName, LogDir, All, Missing), - NewResult = make_one_index_entry1(SuiteName, LogDir, Succ, Fail, + NewResult = make_one_index_entry1(SuiteName, LogDir, Label, Succ, Fail, UserSkip, AutoSkip, NotBuilt, All), {NewResult,Succ,Fail,UserSkip,AutoSkip,NotBuilt}; error -> error end. -make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, +make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> LogFile = filename:join(Link, ?suitelog_name ++ ".html"), CrashDumpName = SuiteName ++ "_erl_crash.dump", @@ -857,6 +857,7 @@ make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, end, ["\n", "\n", + Label, Timestamp, "\n", "\n", @@ -956,7 +957,7 @@ index_header(Label, StartTime) -> "

Common Test Framework Log

", "
HistoryNodeLabelTestsNamesTotal",Node,"",Label,"",NoOfTests,"",TestNamesTrunc,"",timestamp(Dir),"",TotalsStr,"
",SuiteName,"",CrashDumpLink,"",integer_to_list(Success),"",FailStr,"
\n" - "\n", + "\n", "\n" "\n", @@ -965,13 +966,17 @@ index_header(Label, StartTime) -> "\n"]]. all_suites_index_header() -> + {ok,Cwd} = file:get_cwd(), + LogDir = filename:basename(Cwd), + AllRuns = "All test runs in \"" ++ LogDir ++ "\"", [header("Test Results") | ["
\n", - "All Test Runs in this directory\n", + "",AllRuns,"\n", "

\n", "
NameTest Name_Ok" "_Failed
\n" - "\n", + "\n", + "\n", "\n", "\n" @@ -984,14 +989,17 @@ all_suites_index_header() -> "\n"]]. all_runs_header() -> - [header("All test runs in current directory") | + {ok,Cwd} = file:get_cwd(), + LogDir = filename:basename(Cwd), + Title = "All test runs in \"" ++ LogDir ++ "\"", + [header(Title) | ["
NameTest NameLabelTest Run Started_Ok" "_
\n" "\n" "\n" "\n" "\n" - "\n" + "\n" "\n" "\n" @@ -1451,7 +1459,7 @@ make_all_suites_index1(When,AllSuitesLogDirs) -> make_all_suites_index2(IndexName,AllSuitesLogDirs) -> {ok,Index0,_Totals} = make_all_suites_index3(AllSuitesLogDirs, all_suites_index_header(), - 0, 0, 0, 0, 0), + 0, 0, 0, 0, 0, []), Index = [Index0|index_footer()], case force_write_file(IndexName, Index) of ok -> @@ -1461,14 +1469,25 @@ make_all_suites_index2(IndexName,AllSuitesLogDirs) -> end. make_all_suites_index3([{SuiteName,[LastLogDir|OldDirs]}|Rest], - Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt) -> + Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, + Labels) -> [EntryDir|_] = filename:split(LastLogDir), Missing = case file:read_file(filename:join(EntryDir,?missing_suites_info)) of {ok,Bin} -> binary_to_term(Bin); _ -> [] end, - case make_one_index_entry(SuiteName, LastLogDir, {true,OldDirs}, Missing) of + {Label,Labels1} = + case proplists:get_value(EntryDir, Labels) of + undefined -> + case read_totals_file(filename:join(EntryDir,?totals_name)) of + {_,Lbl,_,_} -> {Lbl,[{EntryDir,Lbl}|Labels]}; + _ -> {"-",[{EntryDir,"-"}|Labels]} + end; + Lbl -> + {Lbl,Labels} + end, + case make_one_index_entry(SuiteName, LastLogDir, Label, {true,OldDirs}, Missing) of {Result1,Succ,Fail,USkip,ASkip,NotBuilt} -> %% for backwards compatibility AutoSkip1 = case catch AutoSkip+ASkip of @@ -1477,13 +1496,13 @@ make_all_suites_index3([{SuiteName,[LastLogDir|OldDirs]}|Rest], end, make_all_suites_index3(Rest, [Result|Result1], TotSucc+Succ, TotFail+Fail, UserSkip+USkip, AutoSkip1, - TotNotBuilt+NotBuilt); + TotNotBuilt+NotBuilt,Labels1); error -> make_all_suites_index3(Rest, Result, TotSucc, TotFail, - UserSkip, AutoSkip, TotNotBuilt) + UserSkip, AutoSkip, TotNotBuilt,Labels1) end; make_all_suites_index3([], Result, TotSucc, TotFail, UserSkip, AutoSkip, - TotNotBuilt) -> + TotNotBuilt,_) -> {ok, [Result|total_row(TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt,true)], {TotSucc,TotFail,UserSkip,AutoSkip,TotNotBuilt}}. diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 0b4a508321..2591890898 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1434,14 +1434,17 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad); false -> GrsOrCs1 = - lists:map( + lists:flatmap( %% for now, only flat group defs are allowed as %% start options and test spec terms - fun({Group,TCs}) -> - ct_framework:make_conf(TestDir, Suite, - Group, [], TCs); + fun({all,all}) -> + ct_framework:make_all_conf(TestDir, + Suite, []); + ({Group,TCs}) -> + [ct_framework:make_conf(TestDir, Suite, + Group, [], TCs)]; (TC) -> - TC + [TC] end, GrsOrCs), Do = {TestDir,Suite,GrsOrCs1}, final_tests1(Tests, [Do|Final], Skip, Bad) diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 1aa3a859ab..100e33a940 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -218,7 +218,7 @@ get_skipped_cases(Node,Dir,Suite,Cases) -> case lists:keysearch(all,1,Cases) of {value,{all,{skip,Cmt}}} -> [{{Node,Dir},{Suite,Cmt}}]; - false -> + _ -> get_skipped_cases1(Node,Dir,Suite,Cases) end. diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 899c0bc0c3..7bfb9ffb49 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -111,8 +111,9 @@ init_per_testcase(_TestCase, Config) -> case lists:keysearch(master, 1, Config) of false-> test_server:format("See Common Test logs here:\n\n" - "~s/all_runs.html", - [LogDir,LogDir]); + "~s/all_runs.html\n" + "~s/index.html", + [LogDir,LogDir,LogDir,LogDir]); {value, _}-> test_server:format("See CT Master Test logs here:\n\n" "~s/master_runs.html", diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index c3840d0425..cc66c99ea5 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -72,7 +72,7 @@ all(suite) -> subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, - only_skip]. + testcase_in_multiple_groups]. %%-------------------------------------------------------------------- %% TEST CASES @@ -174,6 +174,9 @@ all_groups(Config) when is_list(Config) -> setup_and_execute(all_groups, TestSpec, Config). + +%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! +%%! NOT WORKING skip_all_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -186,6 +189,8 @@ skip_all_groups(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! +%%! NOT WORKING group(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -209,6 +214,8 @@ skip_group(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here! +%%! NOT WORKING group_all_testcases(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -258,6 +265,9 @@ skip_group_testcase(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% + +%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! +%%! NOT WORKING! Group props disappear from get_suite? topgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -269,6 +279,8 @@ topgroup(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! +%%! NOT WORKING! subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -277,6 +289,8 @@ subgroup(Config) when is_list(Config) -> setup_and_execute(subgroup, TestSpec, Config). +%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! +%%! NOT WORKING! skip_subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -331,6 +345,18 @@ skip_subgroup_testcase(Config) when is_list(Config) -> setup_and_execute(skip_subgroup_testcase, TestSpec, Config). +%%%----------------------------------------------------------------- +%%% + +testcase_in_multiple_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir,groups_12_SUITE,[testcase_1a,testcase_1b]}, + {skip_cases,TestDir,groups_12_SUITE,[testcase_1b],"SKIPPED!"}], + + setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -345,7 +371,9 @@ setup_and_execute(TCName, TestSpec, Config) -> end, {Opts,ERPid} = setup(TestTerms, Config), ok = ct_test_support:run(Opts, Config), - ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + TestSpec1 = [{logdir,proplists:get_value(logdir,Opts)}, + {label,proplists:get_value(label,TestTerms)} | TestSpec], + ok = ct_test_support:run(ct, run_testspec, [TestSpec1], Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(TCName, -- cgit v1.2.3 From afbd6fa341f89211697a3c3f59f888b8247aa7f4 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 1 Jul 2010 15:46:13 +0200 Subject: Add textured backgound to html log files The overview html log files now have a textured background. This default new look can be switched off with the 'basic_html' option. --- lib/common_test/Makefile | 6 +-- lib/common_test/priv/Makefile.in | 20 ++++----- lib/common_test/priv/tile1.jpg | Bin 0 -> 34734 bytes lib/common_test/src/ct_framework.erl | 30 ++++++++++++++ lib/common_test/src/ct_logs.erl | 68 +++++++++++++++++++------------ lib/common_test/src/ct_testspec.erl | 2 + lib/common_test/src/ct_util.erl | 9 +++- lib/test_server/src/test_server_ctrl.erl | 36 +++++++++------- 8 files changed, 116 insertions(+), 55 deletions(-) create mode 100644 lib/common_test/priv/tile1.jpg (limited to 'lib') diff --git a/lib/common_test/Makefile b/lib/common_test/Makefile index e16efd0c9d..aecec1a50d 100644 --- a/lib/common_test/Makefile +++ b/lib/common_test/Makefile @@ -25,12 +25,12 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk # ifeq ($(findstring linux,$(TARGET)),linux) -SUB_DIRECTORIES = doc/src src +SUB_DIRECTORIES = doc/src src priv else ifeq ($(findstring solaris,$(TARGET)),solaris) -SUB_DIRECTORIES = doc/src src +SUB_DIRECTORIES = doc/src src priv else -SUB_DIRECTORIES = doc/src src +SUB_DIRECTORIES = doc/src src priv endif endif diff --git a/lib/common_test/priv/Makefile.in b/lib/common_test/priv/Makefile.in index f144847a29..6372bbc8d5 100644 --- a/lib/common_test/priv/Makefile.in +++ b/lib/common_test/priv/Makefile.in @@ -56,8 +56,9 @@ ifneq ($(findstring win32,$(TARGET)),win32) # # Files # -FILES = vts.tool run_test.in -SCRIPTS = install.sh +FILES = +SCRIPTS = +IMAGES = tile1.jpg # # Rules @@ -83,14 +84,12 @@ include $(ERL_TOP)/make/otp_release_targets.mk ifeq ($(XNIX),true) release_spec: opt - $(INSTALL_DIR) $(RELSYSDIR)/priv/bin - $(INSTALL_SCRIPT) $(SCRIPTS) $(RELSYSDIR) - $(INSTALL_DATA) $(FILES) $(RELSYSDIR)/priv + $(INSTALL_DIR) $(RELSYSDIR)/priv + $(INSTALL_DATA) $(FILES) $(IMAGES) $(RELSYSDIR)/priv else release_spec: opt - $(INSTALL_DIR) $(RELSYSDIR)/priv/bin - $(INSTALL_SCRIPT) $(SCRIPTS) $(RELSYSDIR) - $(INSTALL_DATA) $(FILES) $(RELSYSDIR)/priv + $(INSTALL_DIR) $(RELSYSDIR)/priv + $(INSTALL_DATA) $(FILES) $(IMAGES) $(RELSYSDIR)/priv endif release_docs_spec: @@ -105,6 +104,7 @@ else # Files # FILES = vts.tool +IMAGES = tile1.jpg # # Rules @@ -123,8 +123,8 @@ clean: include $(ERL_TOP)/make/otp_release_targets.mk release_spec: opt - $(INSTALL_DIR) $(RELSYSDIR)/priv/bin - $(INSTALL_DATA) $(FILES) $(RELSYSDIR)/priv + $(INSTALL_DIR) $(RELSYSDIR)/priv + $(INSTALL_DATA) $(FILES) $(IMAGES) $(RELSYSDIR)/priv release_docs_spec: diff --git a/lib/common_test/priv/tile1.jpg b/lib/common_test/priv/tile1.jpg new file mode 100644 index 0000000000..8749383716 Binary files /dev/null and b/lib/common_test/priv/tile1.jpg differ diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 1c8a14574f..17215e5eb5 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -27,6 +27,8 @@ -export([init_tc/3, end_tc/3, get_suite/2, report/2, warn/1]). -export([error_notification/4]). +-export([overview_html_header/1]). + -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). -export([make_all_conf/3, make_conf/5]). @@ -1218,4 +1220,32 @@ add_data_dir(File,Config) when is_list(File) -> File end. +%%%----------------------------------------------------------------- +%%% @spec overview_html_header(TestName) -> Header +overview_html_header(TestName) -> + TestName1 = lists:flatten(io_lib:format("~p", [TestName])), + Label = case application:get_env(common_test, test_label) of + {ok,Lbl} when Lbl =/= undefined -> + "

" ++ Lbl ++ "

\n"; + _ -> + "" + end, + Bgr = case ct_logs:basic_html() of + true -> + ""; + false -> + CTPath = ct_util:get_ct_root(), + TileFile = filename:join(filename:join(filename:join(CTPath), + "priv"),"tile1.jpg"), + " background=\"" ++ TileFile ++ "\"" + end, + + ["\n", + "Test ", TestName1, " results\n", + "\n", + "\n", + "\n", + Label, + "

Results from test ", TestName1, "

\n"]. diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index bb859d3b72..004d0d2540 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -36,7 +36,8 @@ -export([make_all_suites_index/1,make_all_runs_index/1]). %% Logging stuff directly from testcase --export([tc_log/3,tc_print/3,tc_pal/3]). +-export([tc_log/3,tc_print/3,tc_pal/3, + basic_html/0]). %% Simulate logger process for use without ct environment running -export([simulate/0]). @@ -57,7 +58,7 @@ -define(table_color2,"#E4F0FE"). -define(table_color3,"#F0F8FF"). --define(testname_width, 55). +-define(testname_width, 60). -define(abs(Name), filename:absname(Name)). @@ -808,7 +809,7 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, false -> "" end, - {Timestamp,Node,AllInfo} = + {Lbl,Timestamp,Node,AllInfo} = case All of {true,OldRuns} -> [_Prefix,NodeOrDate|_] = string:tokens(Link,"."), @@ -816,20 +817,21 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, 0 -> "-"; _ -> NodeOrDate end, - N = ["\n"], + N = ["\n"], CtRunDir = filename:dirname(filename:dirname(Link)), - T = ["\n"], + L = ["\n"], + T = ["\n"], CtLogFile = filename:join(CtRunDir,?ct_log_name), OldRunsLink = case OldRuns of [] -> "none"; _ -> "Old Runs" end, - A=["\n", - "\n"], - {T,N,A}; + A=["\n", + "\n"], + {L,T,N,A}; false -> - {"","",""} + {"","","",""} end, NotBuiltStr = if NotBuilt == 0 -> @@ -856,8 +858,8 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, {UserSkip+AutoSkip,integer_to_list(UserSkip),ASStr} end, ["\n", - "\n", - Label, + "\n", + Lbl, Timestamp, "\n", "\n", @@ -868,12 +870,14 @@ make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, AllInfo, "\n"]. total_row(Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> - {TimestampCell,AllInfo} = + {Label,TimestampCell,AllInfo} = case All of - true -> - {"\n","\n\n"}; + true -> + {"\n", + "\n", + "\n\n"}; false -> - {"",""} + {"","",""} end, {AllSkip,UserSkipStr,AutoSkipStr} = @@ -883,6 +887,7 @@ total_row(Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> end, ["\n", "", + Label, TimestampCell, "\n", "\n", @@ -1047,19 +1052,28 @@ index_footer() -> footer() -> ["

\n" - "
\n" - "

\n" - "Copyright © ", year(), - " Open Telecom Platform
\n" - "Updated: ", current_time(), "
\n" - "\n" - "

\n" - "\n"]. + "

\n" + "
\n" + "

\n" + "Copyright © ", year(), + " Open Telecom Platform
\n" + "Updated: ", current_time(), "
\n" + "\n" + "\n" + "\n"]. body_tag() -> - "\n". + case basic_html() of + true -> + "\n"; + false -> + CTPath = ct_util:get_ct_root(), + TileFile = filename:join(filename:join(filename:join(CTPath),"priv"),"tile1.jpg"), + "\n" + end. current_time() -> format_time(calendar:local_time()). @@ -1297,7 +1311,7 @@ runentry(Dir, BasicHtml) -> end, Total = TotSucc+TotFail+AllSkip, A = ["

\n", - "\n", + "\n", "\n"], B = if BasicHtml -> ["\n"]; @@ -1318,7 +1332,7 @@ runentry(Dir, BasicHtml) -> end, Index = filename:join(Dir,?index_name), ["\n" - "\n" + "\n" "\n"]. write_totals_file(Name,Label,Logs,Totals) -> diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 100e33a940..1ca60dc92c 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -1045,6 +1045,8 @@ valid_terms() -> {alias,3}, {logdir,2}, {logdir,3}, + {label,2}, + {label,3}, {event_handler,2}, {event_handler,3}, {event_handler,4}, diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl index eddaf4c8b9..86e1482eb0 100644 --- a/lib/common_test/src/ct_util.erl +++ b/lib/common_test/src/ct_util.erl @@ -45,7 +45,7 @@ -export([get_mode/0, create_table/3, read_opts/0]). --export([set_cwd/1, reset_cwd/0]). +-export([set_cwd/1, reset_cwd/0, get_ct_root/0]). -export([parse_table/1]). @@ -619,6 +619,13 @@ get_testdir(Dir, Suite) when is_list(Suite) -> get_testdir(Dir, _) -> get_testdir(Dir, all). +%%%----------------------------------------------------------------- +%%% @spec +%%% +%%% @doc +get_ct_root() -> + [_Ebin|CTPath] = lists:reverse(filename:split(filename:dirname(code:which(?MODULE)))), + lists:reverse(CTPath). %%%----------------------------------------------------------------- %%% @spec diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index a28a73a8ed..38a0b6d94d 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1674,19 +1674,27 @@ do_test_cases(TopCases, SkipCases, TI = get_target_info(), print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]}, {" (with repeated test cases)",[]})]), - test_server_sup:framework_call(report, [tests_start, - {get(test_server_name),N}]), - print(html, - "\n" - "\n" - "\n" - "Test ~p results\n" - "\n" - "\n" - "" - "

Results from test ~p

\n", - [get(test_server_name),get(test_server_name)]), + Test = get(test_server_name), + test_server_sup:framework_call(report, [tests_start,{Test,N}]), + + Header = + case test_server_sup:framework_call(overview_html_header, [Test], "") of + "" -> + TestName = lists:flatten(io_lib:format("~p", [Test])), + ["\n", + "\n", + "\n", + "Test ", TestName, " results\n", + "\n", + "\n", + "", + "

Results from test ", TestName, "

\n"]; + Html -> + ["\n", + "\n" | Html] + end, + print(html, Header, []), print_timestamp(html, "Test started at "), print(html, "

Host:
\n"), @@ -1717,7 +1725,7 @@ do_test_cases(TopCases, SkipCases, [?suitelog_name,?coverlog_name]), print(html,"

~s" "

\n" - "

HistoryNodeLabelTestsNamesTest NamesTotal_Ok" "_",Node1,"",Node1,"",timestamp(CtRunDir),"",Label,"",timestamp(CtRunDir),"CT Log",OldRunsLink,"CT Log",OldRunsLink,"
",SuiteName,"",CrashDumpLink,"",SuiteName,"",CrashDumpLink,"",integer_to_list(Success),"",FailStr,"
       
Total",integer_to_list(Success),"",integer_to_list(Fail),"",Node,"",Label,"",Label,"",NoOfTests,"",TestNamesTrunc,"
",timestamp(Dir),"",TotalsStr,"",timestamp(Dir),"",TotalsStr,"
" + "
" "" "\n", [print_if_known(N, {"Suite contains ~p test cases.\n",[N]}, -- cgit v1.2.3 From 6b51c530111e478c9605cb88bb90178682ffe669 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 1 Jul 2010 20:41:33 +0200 Subject: Improve handling of test case group specifications Still a few things missing: * Documentation * More tests * Fix old broken test cases --- lib/common_test/src/ct_framework.erl | 17 +++++++- lib/common_test/src/ct_run.erl | 22 +++++++--- lib/common_test/src/ct_testspec.erl | 7 ++-- lib/common_test/test/ct_testspec_1_SUITE.erl | 48 +++++++++++++--------- .../groups_1/groups_12_SUITE.erl | 40 +++++++++++++++--- lib/test_server/src/test_server_ctrl.erl | 10 +++-- 6 files changed, 105 insertions(+), 39 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 17215e5eb5..4104115484 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -687,7 +687,20 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> [] -> {error,{invalid_group_spec,Name}}; ConfTests -> - ConfTests + case lists:member(skipped, Props) of + true -> + %% a *subgroup* specified *only* as skipped (and not + %% as an explicit test) should not be returned, or + %% init/end functions for top groups will be executed + case catch proplists:get_value(name, element(2, hd(ConfTests))) of + Name -> % top group + ConfTests; + _ -> + [] + end; + false -> + ConfTests + end end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", @@ -874,7 +887,7 @@ make_all_conf(Mod) -> [] -> {error,{invalid_group_spec,Mod}}; ConfTests -> - ConfTests + [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests] end end. diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 2591890898..c5bfd01642 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1378,15 +1378,17 @@ suite_tuples([]) -> final_tests(Tests, Skip, Bad) -> -%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! -io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]), + %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! + %%! io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]), {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad), + Skip2 = final_skip(Skip1, []), -%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! -io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip1]), - {Tests1,final_skip(Skip1, [])}. + %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! + %%! io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip2]), + + {Tests1,Skip2}. final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) -> @@ -1440,7 +1442,10 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when fun({all,all}) -> ct_framework:make_all_conf(TestDir, Suite, []); - ({Group,TCs}) -> + ({skipped,Group,TCs}) -> + [ct_framework:make_conf(TestDir, Suite, + Group, [skipped], TCs)]; + ({Group,TCs}) -> [ct_framework:make_conf(TestDir, Suite, Group, [], TCs)]; (TC) -> @@ -1453,6 +1458,11 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when final_tests1([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. +final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) -> + SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all), + Skip = {TestDir,Suite,SkipConf,Reason}, + final_skip(Skips, [Skip|Final]); + final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs), Skip = {TestDir,Suite,Conf,Reason}, diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 1ca60dc92c..f5069427a2 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -185,13 +185,13 @@ prepare_cases(Node,Dir,Suite,Cases) -> {[{{Node,Dir},{Suite,all}}],SkipAll}; Skipped -> %% note: this adds a test even if only skip is specified - PrepC = lists:foldr(fun({{G,_}=Group,{skip,_Cmt}}, Acc) when + PrepC = lists:foldr(fun({{G,Cs},{skip,_Cmt}}, Acc) when is_atom(G) -> case lists:keymember(G, 1, Cases) of true -> Acc; false -> - [Group|Acc] + [{skipped,G,Cs}|Acc] end; ({C,{skip,_Cmt}},Acc) -> case lists:member(C,Cases) of @@ -917,7 +917,8 @@ skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case), + Case =/= all -> skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index cc66c99ea5..dc399bfb4c 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -72,6 +72,7 @@ all(suite) -> subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, + sub_skipped_by_top, testcase_in_multiple_groups]. %%-------------------------------------------------------------------- @@ -174,9 +175,6 @@ all_groups(Config) when is_list(Config) -> setup_and_execute(all_groups, TestSpec, Config). - -%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! -%%! NOT WORKING skip_all_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -189,8 +187,6 @@ skip_all_groups(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! -%%! NOT WORKING group(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -206,7 +202,7 @@ skip_group(Config) when is_list(Config) -> TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a, test_group_1b]}, {skip_groups,TestDir,groups_11_SUITE, - [test_group_1b,test_group_2],"SKIPPED!"}], + [test_group_1b,test_group_2,test_group_7],"SKIPPED!"}], setup_and_execute(skip_group, TestSpec, Config). @@ -214,8 +210,6 @@ skip_group(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here! -%%! NOT WORKING group_all_testcases(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -255,7 +249,7 @@ skip_group_testcase(Config) when is_list(Config) -> TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a, {cases,[testcase_1a,testcase_1b]}}, {groups,TestDir,groups_11_SUITE,test_group_1b, - {cases,[testcase_1a,testcase_1b]}}, + {cases,[testcase_1b,testcase_1a]}}, {skip_groups,TestDir,groups_11_SUITE, test_group_1a,{cases,testcase_1b},"SKIPPED!"}, {skip_groups,TestDir,groups_11_SUITE, @@ -266,21 +260,18 @@ skip_group_testcase(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! -%%! NOT WORKING! Group props disappear from get_suite? topgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "groups_1"), - TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}], + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}, + {groups,TestDir,groups_12_SUITE,test_group_4}], setup_and_execute(topgroup, TestSpec, Config). %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! -%%! NOT WORKING! subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -289,15 +280,13 @@ subgroup(Config) when is_list(Config) -> setup_and_execute(subgroup, TestSpec, Config). -%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! -%%! NOT WORKING! skip_subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "groups_1"), - TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]}, + TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_4]}, {skip_groups,TestDir,groups_12_SUITE, - test_group_3,"SKIPPED!"}], + test_group_8,"SKIPPED!"}], setup_and_execute(skip_subgroup, TestSpec, Config). @@ -309,7 +298,9 @@ subgroup_all_testcases(Config) when is_list(Config) -> TestDir = filename:join(DataDir, "groups_1"), TestSpec = [{groups,TestDir,groups_12_SUITE, - test_group_5,{cases,all}}], + test_group_5,{cases,all}}, + {groups,TestDir,groups_12_SUITE, + test_group_3,{cases,all}}], setup_and_execute(subgroup_all_testcases, TestSpec, Config). @@ -331,7 +322,9 @@ subgroup_testcase(Config) when is_list(Config) -> TestDir = filename:join(DataDir, "groups_1"), TestSpec = [{groups,TestDir,groups_12_SUITE, - test_group_7,{cases,testcase_7a}}], + test_group_7,{cases,testcase_7a}}, + {groups,TestDir,groups_12_SUITE, + test_group_3,{cases,testcase_3b}}], setup_and_execute(subgroup_testcase, TestSpec, Config). @@ -348,6 +341,21 @@ skip_subgroup_testcase(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! +%%! Somewhat weird result from this one: +%%! +sub_skipped_by_top(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5}, + {skip_groups,TestDir,groups_12_SUITE,test_group_4,"SKIPPED!"}], + + setup_and_execute(sub_skipped_by_top, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + testcase_in_multiple_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl index dbf1576c76..69c06f9b83 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl @@ -48,9 +48,12 @@ groups() -> testcase_5b]}]}, - {test_group_6, [parallel], [{group, test_group_7}]}, + {test_group_6, [parallel], [{group, test_group_7}, + {group, test_group_8}]}, - {test_group_7, [sequence], [testcase_7a,testcase_7b]} + {test_group_7, [sequence], [testcase_7a,testcase_7b]}, + + {test_group_8, [shuffle,sequence], [testcase_8a,testcase_8b]} ]. all() -> @@ -68,7 +71,8 @@ grs_and_tcs() -> test_group_1a, test_group_1b, test_group_2, test_group_3, test_group_4, test_group_5, - test_group_6, test_group_7 + test_group_6, test_group_7, + test_group_8 ], [ testcase_1a, testcase_1b, testcase_1c, @@ -78,7 +82,8 @@ grs_and_tcs() -> testcase_3a, testcase_3b, testcase_3, testcase_5a, testcase_5b, - testcase_7a, testcase_7b + testcase_7a, testcase_7b, + testcase_8a, testcase_8b ]}. %%-------------------------------------------------------------------- @@ -107,7 +112,9 @@ init_per_group(Group, Config) -> {test_group_4,[{name,test_group_4}]} -> ok; {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; - {test_group_7,[{name,test_group_7},sequence]} -> "sequence" + {test_group_7,[{name,test_group_7},sequence]} -> "sequence"; + {test_group_8,[{shuffle,_},{name,test_group_8},sequence]} -> + "shuffle & sequence" end, {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of @@ -312,3 +319,26 @@ testcase_7b(Config) -> undefined = ?config(testcase_7a,Config), testcase_7b = ?config(testcase_7b,Config), ok. + +testcase_8a() -> + []. +testcase_8a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_8 = ?config(test_group_8,Config), + testcase_8a = ?config(testcase_8a,Config), + ok. +testcase_8b() -> + []. +testcase_8b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_8 = ?config(test_group_8,Config), + undefined = ?config(testcase_8a,Config), + testcase_8b = ?config(testcase_8b,Config), + ok. diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 38a0b6d94d..72f274d63a 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1671,6 +1671,7 @@ do_test_cases(TopCases, SkipCases, put(test_server_case_num, 0), TestSpec = add_init_and_end_per_suite(TestSpec0, undefined, undefined), + TI = get_target_info(), print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]}, {" (with repeated test cases)",[]})]), @@ -4436,7 +4437,7 @@ collect_all_cases(Top, Skip) when is_list(Skip) -> Result = case collect_cases(Top, #cc{mod=[],skip=Skip}) of {ok,Cases,_St} -> Cases; - Other -> Other + Other -> Other end, Result. @@ -4555,8 +4556,8 @@ collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) -> collect_cases(Other, St) -> {error,{bad_subtest_spec,St#cc.mod,Other}}. -collect_case({_Mod,{conf,_,_,_,_}=Conf}, St) -> - collect_cases(Conf, St); +collect_case({Mod,{conf,_,_,_,_}=Conf}, St) -> + collect_case_invoke(Mod, Conf, [], St); collect_case(MFA, St) -> case in_skip_list(MFA, St#cc.skip) of @@ -4593,6 +4594,7 @@ collect_case_invoke(Mod, Case, MFA, St) -> collect_subcases(Mod, Case, MFA, St, Suite) -> case Suite of [] when Case == all -> {ok,[],St}; + [] when element(1, Case) == conf -> {ok,[],St}; [] -> {ok,[MFA],St}; %%%! --- START Kept for backwards compatibilty --- %%%! Requirements are not used @@ -4680,6 +4682,8 @@ in_skip_list({Mod,{conf,Props,InitMF,_CaseList,_FinMF}}, SkipList) -> fun({M,{conf,SProps,_,SCaseList,_},Cmt}) when M == Mod -> case proplists:get_value(name, SProps) of + all -> + [{M,all,Cmt}]; Name -> case SCaseList of all -> -- cgit v1.2.3 From 2c3e05dbf52c1e682bf23bfb06664420433e1774 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Fri, 2 Jul 2010 11:46:53 +0200 Subject: Add documentation for group specifications and for the label option --- lib/common_test/doc/src/run_test.xml | 13 ++- lib/common_test/doc/src/run_test_chapter.xml | 118 +++++++++++++++++++-------- lib/common_test/src/ct.erl | 20 +++-- 3 files changed, 104 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/lib/common_test/doc/src/run_test.xml b/lib/common_test/doc/src/run_test.xml index d609c4287f..2f0a94afba 100644 --- a/lib/common_test/doc/src/run_test.xml +++ b/lib/common_test/doc/src/run_test.xml @@ -45,6 +45,11 @@ flags start an Erlang node prepared for running Common Test in a particular mode.

+

There is an interface function that corresponds to this program, + called ct:run_test/1, for starting Common Test from the Erlang + shell (or an Erlang program). Please see the ct man page for + details.

+

run_test also accepts Erlang emulator flags. These are used when run_test calls erl to start the Erlang node (making it possible to e.g. add directories to the code server path, @@ -72,6 +77,8 @@

it prints all valid start flags to stdout.

+ +
Run tests from command line 
@@ -83,6 +90,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
 	 ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
 	[-stylesheet CSSFile]
@@ -107,6 +115,7 @@
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
+	[-label Label]
 	[-logdir LogDir]
 	[-allow_user_terms]
 	[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]
@@ -128,12 +137,12 @@
     Run tests in web based GUI
     
         run_test -vts [-browser Browser]
+        [-dir TestDir1 TestDir2 .. TestDirN] |
+        [-suite Suite [[-group Group] [-case Case]]]
 	[-config ConfigFile1 ConfigFile2 .. ConfigFileN]
 	[-userconfig CallbackModule1 ConfigString1 and CallbackModule2
          ConfigString2 and .. and CallbackModuleN ConfigStringN]
 	[-decrypt_key Key] | [-decrypt_file KeyFile]
-        [-dir TestDir1 TestDir2 .. TestDirN] |
-        [-suite Suite [[-group Group] [-case Case]]]
 	[-include InclDir1 InclDir2 .. InclDirN]
 	[-no_auto_compile]
 	[-muliply_timetraps Multiplier]
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 207df7f5b5..1efff25f5b 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -21,7 +21,7 @@
 
     
 
-    Running Test Suites
+    Running Tests
     Peter Andersson, Kenneth Lundin
     
     
@@ -130,6 +130,8 @@
     
Other flags that may be used with run_test:

     
       ]]>, specifies where the HTML log files are to be written.
+      ]]>, associates the test run with a name that gets printed
+	in the overview HTML log files.
       -refresh_logs, refreshes the top level HTML index files.
       -vts, start web based GUI (see below).
       -shell, start interactive shell mode (see below).
@@ -335,43 +337,72 @@
     
     Using test specifications
     
-    
The most expressive way to specify what to test is to use a so 
-      called test specification. A test specification is a sequence of 
-      Erlang terms. The terms may be declared in a text file or passed 
-      to the test server at runtime as a list (see run_testspec/1
-      in the manual page for ct). There are two general types 
-      of terms: configuration terms and test specification terms.

-    
With configuration terms it is possible to import configuration 
-      data (similar to run_test -config), specify HTML log 
-      directories (similar to run_test -logdir), give aliases 
-      to test nodes and test directories (to make a specification
-      easier to read and maintain), enable code coverage analysis
-      (see the Code Coverage 
-       Analysis chapter) and specify event_handler plugins 
+    
The most flexible way to specify what to test, is to use a so
+      called test specification. A test specification is a sequence of
+      Erlang terms. The terms may be declared in a text file or passed
+      to the test server at runtime as a list
+      (see run_testspec/1 in the manual page
+      for ct). There are two general types of terms:
+      configuration terms and test specification terms.

+    
With configuration terms it is possible to e.g. label the test
+      run (similar to run_test -label), evaluate arbitrary expressions
+      before starting a test, import configuration
+      data (similar to
+      run_test -config/-userconfig), specify HTML log directories (similar
+      to
+      run_test -logdir), give aliases to test nodes and test
+      directories (to make a specification easier to read and
+      maintain), enable code coverage analysis (see
+      the Code Coverage
+      Analysis chapter) and specify event_handler plugins
       (see the 
-       Event Handling chapter). There is also a term
-       for specifying include directories that should be passed on
-       to the compiler when automatic compilation is performed 
-       (similar to run_test -include, see above).

-    
With test specification terms it is possible to state exactly which
-      tests should run and in which order. A test term specifies either
-      one or more suites or one or more test cases. An arbitrary number of test 
-      terms may be declared in sequence. A test term can also specify one or 
-      more test suites or test cases to be skipped. Skipped suites and cases 
-      are not executed and show up in the HTML test log as SKIPPED.

-
-    
It is not yet possible to specify test case groups in
-      test specifications. This will be supported in a soon upcoming 
-      release.

+      Event Handling chapter). There is also a term for
+      specifying include directories that should be passed on to the
+      compiler when automatic compilation is performed (similar
+      to run_test -include, see above).

+    
With test specification terms it is possible to state exactly
+      which tests should run and in which order. A test term specifies
+      either one or more suites, one or more test case groups, or one
+      or more test cases in a group or suite.

+    
An arbitrary number of test terms may be declared in sequence.
+      Common Test will compile the terms into one or more tests to be
+      performed in one resulting test run. Note that a term that
+      specifies a set of test cases will "swallow" one that only
+      specifies a subset of these cases. E.g. the result of merging
+      one term that specifies that all cases in suite S should be
+      executed, with another term specifying only test case X and Y in
+      S, is a test of all cases in S. However, if a term specifying
+      test case X and Y in S is merged with a term specifying case Z
+      in S, the result is a test of X, Y and Z in S.

+    
A test term can also specify one or more test suites, groups,
+      or test cases to be skipped. Skipped suites, groups and cases
+      are not executed and show up in the HTML test log files as
+      SKIPPED.

+    
When a test case group is specified, the resulting test
+      executes the
+      init_per_group function, followed by all test cases and
+      sub groups (including their configuration functions), and
+      finally the end_per_group function. Also if particular
+      test cases in a group are specified, init_per_group
+      and end_per_group for the group in question are
+      called. If a group which is defined (in Suite:group/0) to
+      be a sub group of another group, is specified (or particular test
+      cases of a sub group are), Common Test will call the configuration
+      functions for the top level groups as well as for the sub group
+      in question (making it possible to pass configuration data all
+      the way from init_per_suite down to the test cases in the
+      sub group).

 
     
Below is the test specification syntax. Test specifications can
-      be used to run tests both in a single test host environment and in 
-      a distributed Common Test environment (Large Scale Testing). The init term,
-      as well as node parameters, are only relevant in the latter (see the
-      Large Scale Testing
-      chapter for information). For details on the event_handler term, see the
-      Event Handling
-      chapter.

+      be used to run tests both in a single test host environment and
+      in a distributed Common Test environment (Large Scale
+      Testing). The node parameters in the init term are only
+      relevant in the latter (see the
+      Large
+      Scale Testing chapter for information). For details on
+      the event_handler term, see the
+      Event
+      Handling chapter.

       
Config terms:

     
       {node, NodeAlias, Node}.
@@ -379,11 +410,17 @@
       {init, InitOptions}.
       {init, [NodeAlias], InitOptions}.
 
+      {label, Label}.
+      {label, NodeRefs, Label}.
+
       {multiply_timetraps, N}.
+      {multiply_timetraps, NodeRefs, N}.
+
       {scale_timetraps, Bool}.
+      {scale_timetraps, NodeRefs, Bool}.
  
       {cover, CoverSpecFile}.
-      {cover, NodeRef, CoverSpecFile}.
+      {cover, NodeRefs, CoverSpecFile}.
       
       {include, IncludeDirs}.
       {include, NodeRefs, IncludeDirs}.
@@ -409,6 +446,12 @@
       {suites, DirRef, Suites}.                                
       {suites, NodeRefs, DirRef, Suites}.
       
+      {groups, DirRef, Suite, Groups}.
+      {groups, NodeRefsDirRef, Suite, Groups}.
+
+      {groups, DirRef, Suite, Group, {cases,Cases}}.
+      {groups, NodeRefsDirRef, Suite, Group, {cases,Cases}}.
+
       {cases, DirRef, Suite, Cases}.                           
       {cases, NodeRefs, DirRef, Suite, Cases}.
 
@@ -437,6 +480,9 @@
       InitArgs      = [term()]
       DirRef        = DirAlias | Dir
       Suites        = atom() | [atom()] | all
+      Suite         = atom()
+      Groups        = atom() | [atom()] | all
+      Group         = atom()
       Cases         = atom() | [atom()] | all
       Comment       = string() | ""
     

diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl
index 0d82a86e7d..8ae175f10d 100644
--- a/lib/common_test/src/ct.erl
+++ b/lib/common_test/src/ct.erl
@@ -138,10 +138,10 @@ run(TestDirs) ->
 %%%-----------------------------------------------------------------
 %%% @spec run_test(Opts) -> Result
 %%%   Opts = [OptTuples]
-%%%   OptTuples = {config,CfgFiles} | {dir,TestDirs} | {suite,Suites} |
-%%%               {userconfig, UserConfig} |
-%%%               {testcase,Cases} | {group,Groups} | {spec,TestSpecs} |
-%%%               {allow_user_terms,Bool} | {logdir,LogDir} | 
+%%%   OptTuples = {dir,TestDirs} | {suite,Suites} | {group,Groups} |
+%%%               {testcase,Cases} | {spec,TestSpecs} | {label,Label} |
+%%%               {config,CfgFiles} | {userconfig, UserConfig} |
+%%%               {allow_user_terms,Bool} | {logdir,LogDir} |
 %%%               {silent_connections,Conns} | {stylesheet,CSSFile} |
 %%%               {cover,CoverSpecFile} | {step,StepOpts} |
 %%%               {event_handler,EventHandlers} | {include,InclDirs} |
@@ -149,15 +149,16 @@ run(TestDirs) ->
 %%%               {repeat,N} | {duration,DurTime} | {until,StopTime} |
 %%%               {force_stop,Bool} | {decrypt,DecryptKeyOrFile} |
 %%%               {refresh_logs,LogDir} | {basic_html,Bool}
-%%%   CfgFiles = [string()] | string()
-%%%   UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
-%%%   CallbackMod = atom()
-%%%   CfgStrings = [string()] | string()
 %%%   TestDirs = [string()] | string()
 %%%   Suites = [string()] | string()
 %%%   Cases = [atom()] | atom()
 %%%   Groups = [atom()] | atom()
 %%%   TestSpecs = [string()] | string()
+%%%   Label = string() | atom()
+%%%   CfgFiles = [string()] | string()
+%%%   UserConfig = [{CallbackMod,CfgStrings}] | {CallbackMod,CfgStrings}
+%%%   CallbackMod = atom()
+%%%   CfgStrings = [string()] | string()
 %%%   LogDir = string()
 %%%   Conns = all | [atom()]
 %%%   CSSFile = string()
@@ -177,7 +178,8 @@ run(TestDirs) ->
 %%%   DecryptFile = string()
 %%%   Result = [TestResult] | {error,Reason}
 %%% @doc Run tests as specified by the combination of options in Opts.
-%%% The options are the same as those used with the run_test program.
+%%% The options are the same as those used with the
+%%% run_test program.
 %%% Note that here a TestDir can be used to point out the path to 
 %%% a Suite. Note also that the option testcase
 %%% corresponds to the -case option in the run_test 
-- 
cgit v1.2.3


From ab61ffca8dda3ae387e479bb5ef4f40bd22cc6e9 Mon Sep 17 00:00:00 2001
From: Peter Andersson 
Date: Fri, 2 Jul 2010 12:01:26 +0200
Subject: Bump version numbers for common_test and test_server

---
 lib/common_test/vsn.mk | 2 +-
 lib/test_server/vsn.mk | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/common_test/vsn.mk b/lib/common_test/vsn.mk
index 2947c6a152..413ef21df3 100644
--- a/lib/common_test/vsn.mk
+++ b/lib/common_test/vsn.mk
@@ -1,3 +1,3 @@
-COMMON_TEST_VSN = 1.5
+COMMON_TEST_VSN = 1.5.1
 
 
diff --git a/lib/test_server/vsn.mk b/lib/test_server/vsn.mk
index ee3c957d59..4c3df28814 100644
--- a/lib/test_server/vsn.mk
+++ b/lib/test_server/vsn.mk
@@ -1,2 +1,2 @@
-TEST_SERVER_VSN = 3.4
+TEST_SERVER_VSN = 3.4.1
 
-- 
cgit v1.2.3


From 311ed3490b2b9dc93433827a3941c14e40f41a2e Mon Sep 17 00:00:00 2001
From: Peter Andersson 
Date: Mon, 5 Jul 2010 23:38:58 +0200
Subject: Fix bug that crashes common_test when running cover

---
 lib/common_test/src/ct_framework.erl |  5 ++---
 lib/common_test/src/ct_logs.erl      |  4 ++--
 lib/common_test/src/ct_util.erl      | 10 +---------
 3 files changed, 5 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 4104115484..f2ca023cff 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1247,9 +1247,8 @@ overview_html_header(TestName) ->
 	      true ->
 		  "";
 	      false ->
-		  CTPath = ct_util:get_ct_root(),
-		  TileFile = filename:join(filename:join(filename:join(CTPath),
-							 "priv"),"tile1.jpg"),
+		  CTPath = code:lib_dir(common_test),
+		  TileFile = filename:join(filename:join(CTPath,"priv"),"tile1.jpg"),
 		  " background=\"" ++ TileFile ++ "\""
 	  end,
 
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index 004d0d2540..f8ae7202e6 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -1069,8 +1069,8 @@ body_tag() ->
 	    "\n";
 	false ->
-	    CTPath = ct_util:get_ct_root(),
-	    TileFile = filename:join(filename:join(filename:join(CTPath),"priv"),"tile1.jpg"),
+	    CTPath = code:lib_dir(common_test),
+	    TileFile = filename:join(filename:join(CTPath,"priv"),"tile1.jpg"),
 	    "\n"
     end.
diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl
index 86e1482eb0..0a434666fa 100644
--- a/lib/common_test/src/ct_util.erl
+++ b/lib/common_test/src/ct_util.erl
@@ -45,7 +45,7 @@
 
 -export([get_mode/0, create_table/3, read_opts/0]).
 
--export([set_cwd/1, reset_cwd/0, get_ct_root/0]).
+-export([set_cwd/1, reset_cwd/0]).
 
 -export([parse_table/1]).
 
@@ -619,14 +619,6 @@ get_testdir(Dir, Suite) when is_list(Suite) ->
 get_testdir(Dir, _) ->
     get_testdir(Dir, all).
 
-%%%-----------------------------------------------------------------
-%%% @spec
-%%%
-%%% @doc
-get_ct_root() ->
-    [_Ebin|CTPath] = lists:reverse(filename:split(filename:dirname(code:which(?MODULE)))),
-    lists:reverse(CTPath).
-
 %%%-----------------------------------------------------------------
 %%% @spec 
 %%%
-- 
cgit v1.2.3
NumModuleCaseLogTimeResultComment