From 79851b599df5caf5e2101d834a843e3837c6a6ae Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Tue, 29 Jun 2010 18:03:09 +0200 Subject: Improve handling of test case group specifications --- lib/common_test/src/ct_framework.erl | 28 ++++++++++++++- lib/common_test/src/ct_logs.erl | 51 +++++++++++++++++++--------- lib/common_test/src/ct_run.erl | 13 ++++--- lib/common_test/src/ct_testspec.erl | 2 +- lib/common_test/test/ct_test_support.erl | 5 +-- lib/common_test/test/ct_testspec_1_SUITE.erl | 32 +++++++++++++++-- 6 files changed, 104 insertions(+), 27 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 25915b9db2..1c8a14574f 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -29,7 +29,7 @@ -export([error_in_suite/1, ct_init_per_group/2, ct_end_per_group/2]). --export([make_conf/5]). +-export([make_all_conf/3, make_conf/5]). -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -850,6 +850,32 @@ expand(Mod, Name, Defs) -> throw({error,list_to_atom(E)}) end. +make_all_conf(Dir, Mod, _Props) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_all_conf(Mod). + +make_all_conf(Mod) -> + case catch apply(Mod, groups, []) of + {'EXIT',_} -> + {error,{invalid_group_definition,Mod}}; + GroupDefs when is_list(GroupDefs) -> + case catch find_groups(Mod, all, all, GroupDefs) of + {error,_} = Error -> + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Error properly + [{?MODULE,error_in_suite,[[Error]]}]; + [] -> + {error,{invalid_group_spec,Mod}}; + ConfTests -> + ConfTests + end + end. + make_conf(Dir, Mod, Name, Props, TestSpec) -> case code:is_loaded(Mod) of false -> diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 3ed0ea5c40..bb859d3b72 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -57,7 +57,7 @@ -define(table_color2,"#E4F0FE"). -define(table_color3,"#F0F8FF"). --define(testname_width, 70). +-define(testname_width, 55). -define(abs(Name), filename:absname(Name)). @@ -766,7 +766,7 @@ make_last_run_index([Name|Rest], Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, Missing); LastLogDir -> SuiteName = filename:rootname(filename:basename(Name)), - case make_one_index_entry(SuiteName, LastLogDir, false, Missing) of + case make_one_index_entry(SuiteName, LastLogDir, "-", false, Missing) of {Result1,Succ,Fail,USkip,ASkip,NotBuilt} -> %% for backwards compatibility AutoSkip1 = case catch AutoSkip+ASkip of @@ -785,18 +785,18 @@ make_last_run_index([], Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuil {ok, [Result|total_row(TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, false)], {TotSucc,TotFail,UserSkip,AutoSkip,TotNotBuilt}}. -make_one_index_entry(SuiteName, LogDir, All, Missing) -> +make_one_index_entry(SuiteName, LogDir, Label, All, Missing) -> case count_cases(LogDir) of {Succ,Fail,UserSkip,AutoSkip} -> NotBuilt = not_built(SuiteName, LogDir, All, Missing), - NewResult = make_one_index_entry1(SuiteName, LogDir, Succ, Fail, + NewResult = make_one_index_entry1(SuiteName, LogDir, Label, Succ, Fail, UserSkip, AutoSkip, NotBuilt, All), {NewResult,Succ,Fail,UserSkip,AutoSkip,NotBuilt}; error -> error end. -make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, +make_one_index_entry1(SuiteName, Link, Label, Success, Fail, UserSkip, AutoSkip, NotBuilt, All) -> LogFile = filename:join(Link, ?suitelog_name ++ ".html"), CrashDumpName = SuiteName ++ "_erl_crash.dump", @@ -857,6 +857,7 @@ make_one_index_entry1(SuiteName, Link, Success, Fail, UserSkip, AutoSkip, end, ["\n", "",SuiteName,"",CrashDumpLink,"\n", + Label, Timestamp, "",integer_to_list(Success),"\n", "",FailStr,"\n", @@ -956,7 +957,7 @@ index_header(Label, StartTime) -> "

Common Test Framework Log

", "\n" - "\n", + "\n", "\n" "\n", @@ -965,13 +966,17 @@ index_header(Label, StartTime) -> "\n"]]. all_suites_index_header() -> + {ok,Cwd} = file:get_cwd(), + LogDir = filename:basename(Cwd), + AllRuns = "All test runs in \"" ++ LogDir ++ "\"", [header("Test Results") | ["
\n", - "All Test Runs in this directory\n", + "",AllRuns,"\n", "

\n", "
NameTest Name_Ok" "_Failed
\n" - "\n", + "\n", + "\n", "\n", "\n" @@ -984,14 +989,17 @@ all_suites_index_header() -> "\n"]]. all_runs_header() -> - [header("All test runs in current directory") | + {ok,Cwd} = file:get_cwd(), + LogDir = filename:basename(Cwd), + Title = "All test runs in \"" ++ LogDir ++ "\"", + [header(Title) | ["
NameTest NameLabelTest Run Started_Ok" "_
\n" "\n" "\n" "\n" "\n" - "\n" + "\n" "\n" "\n" @@ -1451,7 +1459,7 @@ make_all_suites_index1(When,AllSuitesLogDirs) -> make_all_suites_index2(IndexName,AllSuitesLogDirs) -> {ok,Index0,_Totals} = make_all_suites_index3(AllSuitesLogDirs, all_suites_index_header(), - 0, 0, 0, 0, 0), + 0, 0, 0, 0, 0, []), Index = [Index0|index_footer()], case force_write_file(IndexName, Index) of ok -> @@ -1461,14 +1469,25 @@ make_all_suites_index2(IndexName,AllSuitesLogDirs) -> end. make_all_suites_index3([{SuiteName,[LastLogDir|OldDirs]}|Rest], - Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt) -> + Result, TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt, + Labels) -> [EntryDir|_] = filename:split(LastLogDir), Missing = case file:read_file(filename:join(EntryDir,?missing_suites_info)) of {ok,Bin} -> binary_to_term(Bin); _ -> [] end, - case make_one_index_entry(SuiteName, LastLogDir, {true,OldDirs}, Missing) of + {Label,Labels1} = + case proplists:get_value(EntryDir, Labels) of + undefined -> + case read_totals_file(filename:join(EntryDir,?totals_name)) of + {_,Lbl,_,_} -> {Lbl,[{EntryDir,Lbl}|Labels]}; + _ -> {"-",[{EntryDir,"-"}|Labels]} + end; + Lbl -> + {Lbl,Labels} + end, + case make_one_index_entry(SuiteName, LastLogDir, Label, {true,OldDirs}, Missing) of {Result1,Succ,Fail,USkip,ASkip,NotBuilt} -> %% for backwards compatibility AutoSkip1 = case catch AutoSkip+ASkip of @@ -1477,13 +1496,13 @@ make_all_suites_index3([{SuiteName,[LastLogDir|OldDirs]}|Rest], end, make_all_suites_index3(Rest, [Result|Result1], TotSucc+Succ, TotFail+Fail, UserSkip+USkip, AutoSkip1, - TotNotBuilt+NotBuilt); + TotNotBuilt+NotBuilt,Labels1); error -> make_all_suites_index3(Rest, Result, TotSucc, TotFail, - UserSkip, AutoSkip, TotNotBuilt) + UserSkip, AutoSkip, TotNotBuilt,Labels1) end; make_all_suites_index3([], Result, TotSucc, TotFail, UserSkip, AutoSkip, - TotNotBuilt) -> + TotNotBuilt,_) -> {ok, [Result|total_row(TotSucc, TotFail, UserSkip, AutoSkip, TotNotBuilt,true)], {TotSucc,TotFail,UserSkip,AutoSkip,TotNotBuilt}}. diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 0b4a508321..2591890898 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1434,14 +1434,17 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when final_tests1(Tests, [{TestDir,Suite,all}|Final], Skip1, Bad); false -> GrsOrCs1 = - lists:map( + lists:flatmap( %% for now, only flat group defs are allowed as %% start options and test spec terms - fun({Group,TCs}) -> - ct_framework:make_conf(TestDir, Suite, - Group, [], TCs); + fun({all,all}) -> + ct_framework:make_all_conf(TestDir, + Suite, []); + ({Group,TCs}) -> + [ct_framework:make_conf(TestDir, Suite, + Group, [], TCs)]; (TC) -> - TC + [TC] end, GrsOrCs), Do = {TestDir,Suite,GrsOrCs1}, final_tests1(Tests, [Do|Final], Skip, Bad) diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 1aa3a859ab..100e33a940 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -218,7 +218,7 @@ get_skipped_cases(Node,Dir,Suite,Cases) -> case lists:keysearch(all,1,Cases) of {value,{all,{skip,Cmt}}} -> [{{Node,Dir},{Suite,Cmt}}]; - false -> + _ -> get_skipped_cases1(Node,Dir,Suite,Cases) end. diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 899c0bc0c3..7bfb9ffb49 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -111,8 +111,9 @@ init_per_testcase(_TestCase, Config) -> case lists:keysearch(master, 1, Config) of false-> test_server:format("See Common Test logs here:\n\n" - "~s/all_runs.html", - [LogDir,LogDir]); + "~s/all_runs.html\n" + "~s/index.html", + [LogDir,LogDir,LogDir,LogDir]); {value, _}-> test_server:format("See CT Master Test logs here:\n\n" "~s/master_runs.html", diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index c3840d0425..cc66c99ea5 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -72,7 +72,7 @@ all(suite) -> subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, - only_skip]. + testcase_in_multiple_groups]. %%-------------------------------------------------------------------- %% TEST CASES @@ -174,6 +174,9 @@ all_groups(Config) when is_list(Config) -> setup_and_execute(all_groups, TestSpec, Config). + +%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! +%%! NOT WORKING skip_all_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -186,6 +189,8 @@ skip_all_groups(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! +%%! NOT WORKING group(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -209,6 +214,8 @@ skip_group(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here! +%%! NOT WORKING group_all_testcases(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -258,6 +265,9 @@ skip_group_testcase(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% + +%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! +%%! NOT WORKING! Group props disappear from get_suite? topgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -269,6 +279,8 @@ topgroup(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! +%%! NOT WORKING! subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -277,6 +289,8 @@ subgroup(Config) when is_list(Config) -> setup_and_execute(subgroup, TestSpec, Config). +%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! +%%! NOT WORKING! skip_subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -331,6 +345,18 @@ skip_subgroup_testcase(Config) when is_list(Config) -> setup_and_execute(skip_subgroup_testcase, TestSpec, Config). +%%%----------------------------------------------------------------- +%%% + +testcase_in_multiple_groups(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{cases,TestDir,groups_12_SUITE,[testcase_1a,testcase_1b]}, + {skip_cases,TestDir,groups_12_SUITE,[testcase_1b],"SKIPPED!"}], + + setup_and_execute(testcase_in_multiple_groups, TestSpec, Config). + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS %%%----------------------------------------------------------------- @@ -345,7 +371,9 @@ setup_and_execute(TCName, TestSpec, Config) -> end, {Opts,ERPid} = setup(TestTerms, Config), ok = ct_test_support:run(Opts, Config), - ok = ct_test_support:run(ct, run_testspec, [TestSpec], Config), + TestSpec1 = [{logdir,proplists:get_value(logdir,Opts)}, + {label,proplists:get_value(label,TestTerms)} | TestSpec], + ok = ct_test_support:run(ct, run_testspec, [TestSpec1], Config), Events = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(TCName, -- cgit v1.2.3
HistoryNodeLabelTestsNamesTest NamesTotal_Ok" "_