From 3e79e2da66002bd998b67ea3f75955a5bbd7348e Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 16 Apr 2015 11:20:37 +0200 Subject: Improve error reports in log when suite compilation fails --- lib/common_test/src/ct_logs.erl | 9 ++++++++- lib/test_server/src/test_server_ctrl.erl | 18 +++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index 4d5a75d354..7c8c720e13 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -2054,6 +2054,13 @@ runentry(Dir, Totals={Node,Label,Logs, ?testname_width-3)), lists:flatten(io_lib:format("~ts...",[Trunc])) end, + TotMissingStr = + if NotBuilt > 0 -> + ["", + integer_to_list(NotBuilt),""]; + true -> + integer_to_list(NotBuilt) + end, Total = TotSucc+TotFail+AllSkip, A = xhtml(["",Node, "\n", @@ -2073,7 +2080,7 @@ runentry(Dir, Totals={Node,Label,Logs, "",TotFailStr,"\n", "",integer_to_list(AllSkip), " (",UserSkipStr,"/",AutoSkipStr,")\n", - "",integer_to_list(NotBuilt),"\n"], + "",TotMissingStr,"\n"], TotalsStr = A++B++C, XHTML = [xhtml("\n", ["\n"]), diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index bef0658b6d..d0c8a1ebe8 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -4776,17 +4776,25 @@ collect_case_subcases(Mod, Case, SubCases, St0, Mode) -> collect_files(Dir, Pattern, St, Mode) -> {ok,Cwd} = file:get_cwd(), Dir1 = filename:join(Cwd, Dir), - Wc = filename:join([Dir1,Pattern++code:objfile_extension()]), + Wc = filename:join([Dir1,Pattern++"{.erl,"++code:objfile_extension()++"}"]), case catch filelib:wildcard(Wc) of {'EXIT', Reason} -> io:format("Could not collect files: ~p~n", [Reason]), {error,{collect_fail,Dir,Pattern}}; - Mods0 -> - Mods = [{path_to_module(Mod),all} || Mod <- lists:sort(Mods0)], - collect_cases(Mods, St, Mode) + Files -> + %% convert to module names and remove duplicates + Mods = lists:foldl(fun(File, Acc) -> + Mod = fullname_to_mod(File), + case lists:member(Mod, Acc) of + true -> Acc; + false -> [Mod | Acc] + end + end, [], Files), + Tests = [{Mod,all} || Mod <- lists:sort(Mods)], + collect_cases(Tests, St, Mode) end. -path_to_module(Path) when is_list(Path) -> +fullname_to_mod(Path) when is_list(Path) -> %% If this is called with a binary, then we are probably in +fnu %% mode and have found a beam file with name encoded as latin1. We %% will let this crash since it can not work to load such a module -- cgit v1.2.3 From 985215ccba444132fb8e01e72968493402b976a8 Mon Sep 17 00:00:00 2001 From: Peter Andersson Date: Thu, 16 Apr 2015 13:38:34 +0200 Subject: Add missing events and hook function calls --- lib/common_test/src/ct_framework.erl | 18 +++++++- lib/common_test/test/ct_auto_compile_SUITE.erl | 48 +++++++++++++--------- lib/common_test/test/ct_test_server_if_1_SUITE.erl | 25 ++++++----- 3 files changed, 57 insertions(+), 34 deletions(-) (limited to 'lib') diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index ea3d7c8218..0878e97f81 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -113,6 +113,7 @@ init_tc1(?MODULE,_,error_in_suite,[Config0]) when is_list(Config0) -> ct_event:notify(#event{name=tc_start, node=node(), data={?MODULE,error_in_suite}}), + ct_suite_init(?MODULE, error_in_suite, [], Config0), case ?val(error, Config0) of undefined -> {fail,"unknown_error_in_suite"}; @@ -635,7 +636,20 @@ try_set_default(Name,Key,Info,Where) -> end_tc(Mod, Fun, Args) -> %% Have to keep end_tc/3 for backwards compatibility issues end_tc(Mod, Fun, Args, '$end_tc_dummy'). -end_tc(?MODULE,error_in_suite,_, _) -> % bad start! +end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) -> + %% this clause gets called if CT has encountered a suite that + %% can't be executed + FinalNotify = + case ct_hooks:end_tc(?MODULE, error_in_suite, Args, Result, Return) of + '$ct_no_change' -> + Result; + HookResult -> + HookResult + end, + Event = #event{name=tc_done, + node=node(), + data={?MODULE,error_in_suite,tag(FinalNotify)}}, + ct_event:sync_notify(Event), ok; end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) -> end_tc(Mod,Func,TCPid,Result,Args,Return); @@ -1293,6 +1307,8 @@ report(What,Data) -> end, ct_logs:unregister_groupleader(ReportingPid), case {Func,Result} of + {error_in_suite,_} when Suite == ?MODULE -> + ok; {init_per_suite,_} -> ok; {end_per_suite,_} -> diff --git a/lib/common_test/test/ct_auto_compile_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE.erl index cc546ed30d..3e4da31ab4 100644 --- a/lib/common_test/test/ct_auto_compile_SUITE.erl +++ b/lib/common_test/test/ct_auto_compile_SUITE.erl @@ -108,6 +108,8 @@ ac_spec(Config) when is_list(Config) -> PrivDir = ?config(priv_dir, Config), file:copy(filename:join(DataDir, "bad_SUITE.erl"), filename:join(PrivDir, "bad_SUITE.erl")), + Suite = filename:join(DataDir, "dummy_SUITE"), + compile:file(Suite, [{outdir,PrivDir}]), TestSpec = [{label,ac_spec}, {auto_compile,false}, {suites,PrivDir,all}], @@ -160,28 +162,34 @@ events_to_check(Test, N) -> test_events(ac_flag) -> [ - {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {ct_test_support_eh,start_info,{1,1,3}}, - {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, - {ct_test_support_eh,test_stats,{1,1,{1,0}}}, - {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}}, - {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}}, - {ct_test_support_eh,stop_logging,[]} + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,3}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed,{error,'bad_SUITE can not be compiled or loaded'}}}}, + {?eh,tc_start,{dummy_SUITE,init_per_suite}}, + {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{dummy_SUITE,end_per_suite}}, + {?eh,tc_done,{dummy_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} ]; test_events(ac_spec) -> [ - {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}}, - {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {ct_test_support_eh,start_info,{1,1,3}}, - {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}}, - {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, - {ct_test_support_eh,test_stats,{1,1,{1,0}}}, - {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}}, - {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}}, - {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}}, - {ct_test_support_eh,stop_logging,[]} + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,3}}, + {?eh,tc_start,{ct_framework,error_in_suite}}, + {?eh,tc_done,{ct_framework,error_in_suite, + {failed,{error,'bad_SUITE can not be compiled or loaded'}}}}, + {?eh,tc_start,{dummy_SUITE,init_per_suite}}, + {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}}, + {?eh,test_stats,{1,1,{1,0}}}, + {?eh,tc_start,{dummy_SUITE,end_per_suite}}, + {?eh,tc_done,{dummy_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index b6ef3062d4..214cb60c0d 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -236,14 +236,13 @@ test_events(ts_if_1) -> {ts_if_2_SUITE,end_per_suite, {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}}, - {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{2,6,{4,7}}}, - + {?eh,tc_done,{ct_framework,error_in_suite, + {failed,{error,'ts_if_3_SUITE:all/0 is missing'}}}}, {?eh,tc_start,{ct_framework,error_in_suite}}, - {?eh,test_stats,{2,7,{4,7}}}, - + {?eh,tc_done,{ct_framework,error_in_suite, + {failed,{error,'Bad return value from ts_if_4_SUITE:all/0'}}}}, {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}}, {?eh,tc_done,{ts_if_5_SUITE,init_per_suite, @@ -252,7 +251,7 @@ test_events(ts_if_1) -> {?eh,tc_auto_skip, {ts_if_5_SUITE,my_test_case, {require_failed_in_suite0,{not_available,undef_variable}}}}, - {?eh,test_stats,{2,7,{4,8}}}, + {?eh,test_stats,{2,5,{4,8}}}, {?eh,tc_auto_skip, {ts_if_5_SUITE,end_per_suite, {require_failed_in_suite0,{not_available,undef_variable}}}}, @@ -264,7 +263,7 @@ test_events(ts_if_1) -> {?eh,tc_auto_skip, {ts_if_6_SUITE,tc1, {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, - {?eh,test_stats,{2,7,{4,9}}}, + {?eh,test_stats,{2,5,{4,9}}}, {?eh,tc_auto_skip, {ct_framework,end_per_suite, {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}}, @@ -274,13 +273,13 @@ test_events(ts_if_1) -> {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_done, {ts_if_7_SUITE,tc1,{auto_skipped,{testcase0_failed,bad_return_value}}}}, - {?eh,test_stats,{2,7,{4,10}}}, + {?eh,test_stats,{2,5,{4,10}}}, {?eh,tc_done,{ts_if_7_SUITE, {init_per_group,g1,[]}, {auto_skipped,{group0_failed,bad_return_value}}}}, {?eh,tc_auto_skip, {ts_if_7_SUITE,{tc2,g1},{group0_failed,bad_return_value}}}, - {?eh,test_stats,{2,7,{4,11}}}, + {?eh,test_stats,{2,5,{4,11}}}, {?eh,tc_auto_skip, {ts_if_7_SUITE,{end_per_group,g1},{group0_failed,bad_return_value}}}, @@ -288,7 +287,7 @@ test_events(ts_if_1) -> {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_done,{ts_if_7_SUITE,tc2, {auto_skipped,{testcase0_failed,bad_return_value}}}}, - {?eh,test_stats,{2,7,{4,12}}}, + {?eh,test_stats,{2,5,{4,12}}}, {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}}, {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}], @@ -300,17 +299,17 @@ test_events(ts_if_1) -> {?eh,tc_done,{ct_framework,init_per_suite,ok}}, {?eh,tc_start,{ts_if_8_SUITE,tc1}}, {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}}, - {?eh,test_stats,{2,8,{4,12}}}, + {?eh,test_stats,{2,6,{4,12}}}, {?eh,tc_start,{ct_framework,end_per_suite}}, {?eh,tc_done,{ct_framework,end_per_suite,ok}}, {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}}, - {?eh,test_stats,{2,8,{5,12}}}, + {?eh,test_stats,{2,6,{5,12}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}}, {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}}, - {?eh,test_stats,{2,8,{6,12}}}, + {?eh,test_stats,{2,6,{6,12}}}, {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}}, {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}}, -- cgit v1.2.3